Sync standard object metadata (#2807)

* Sync standard object metadata

* remove debug logging

* remove unused func

* fix comments

* fix empty objectsToDelete list

server/package.json

@@ -84,6 +84,7 @@
     "lodash.merge": "^4.6.2",
     "lodash.snakecase": "^4.1.1",
     "lodash.upperfirst": "^4.3.1",
+    "microdiff": "^1.3.2",
     "nest-commander": "^3.12.0",
     "passport": "^0.6.0",
     "passport-google-oauth20": "^2.0.0",

server/src/database/typeorm-seeds/metadata/field-metadata/view-sort.ts

@@ -122,13 +122,13 @@ export const seedViewSortFieldMetadata = async (
         isCustom: false,
         workspaceId: SeedWorkspaceId,
         isActive: true,
-        type: FieldMetadataType.UUID,
+        type: FieldMetadataType.RELATION,
-        name: 'viewId',
+        name: 'view',
-        label: 'View Id',
+        label: 'View',
         targetColumnMap: {},
         description: 'View Sort related view',
         icon: 'IconLayoutCollage',
-        isNullable: false,
+        isNullable: true,
         isSystem: false,
         defaultValue: undefined,
       },

server/src/database/typeorm-seeds/metadata/index.ts

@@ -23,6 +23,7 @@ import { seedPipelineStepRelationMetadata } from 'src/database/typeorm-seeds/met
 import { seedPersonRelationMetadata } from 'src/database/typeorm-seeds/metadata/relation-metadata/person';
 import { seedWorkspaceMemberRelationMetadata } from 'src/database/typeorm-seeds/metadata/relation-metadata/workspace-member';
 import { seedDataSource } from 'src/database/typeorm-seeds/metadata/data-source';
+import { seedWebhookFieldMetadata } from 'src/database/typeorm-seeds/metadata/field-metadata/webhook';
 
 export const seedMetadataSchema = async (workspaceDataSource: DataSource) => {
   const schemaName = 'metadata';
@@ -33,6 +34,7 @@ export const seedMetadataSchema = async (workspaceDataSource: DataSource) => {
   await seedActivityFieldMetadata(workspaceDataSource, schemaName);
   await seedApiKeyFieldMetadata(workspaceDataSource, schemaName);
   await seedAttachmentFieldMetadata(workspaceDataSource, schemaName);
+  await seedWebhookFieldMetadata(workspaceDataSource, schemaName);
   await seedCommentFieldMetadata(workspaceDataSource, schemaName);
   await seedCompanyFieldMetadata(workspaceDataSource, schemaName);
   await seedFavoriteFieldMetadata(workspaceDataSource, schemaName);

server/src/metadata/field-metadata/utils/generate-target-column-map.util.ts

@@ -52,7 +52,8 @@ export function generateTargetColumnMap(
         firstName: `${columnName}FirstName`,
         lastName: `${columnName}LastName`,
       };
-
+    case FieldMetadataType.RELATION:
+      return {};
     default:
       throw new BadRequestException(`Unknown type ${type}`);
   }

server/src/workspace/workspace-manager/commands/sync-workspace-metadata.command.ts

@@ -31,7 +31,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
       );
 
     // TODO: This solution could be improved, using a diff for example, we should not have to delete all metadata and recreate them.
-    await this.workspaceManagerService.resetStandardObjectsAndFieldsMetadata(
+    await this.workspaceManagerService.syncStandardObjectsAndFieldsMetadata(
       dataSourceMetadata.id,
       options.workspaceId,
     );

server/src/workspace/workspace-manager/decorators/metadata.decorator.ts

@@ -0,0 +1,103 @@
+import camelCase from 'lodash.camelcase';
+import 'reflect-metadata';
+
+import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface';
+
+import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
+import { generateTargetColumnMap } from 'src/metadata/field-metadata/utils/generate-target-column-map.util';
+
+export type FieldMetadataDecorator = {
+  type: FieldMetadataType;
+  label: string;
+  description?: string | null;
+  icon?: string | null;
+  defaultValue?: FieldMetadataDefaultValue | null;
+};
+
+export type ObjectMetadataDecorator = {
+  namePlural: string;
+  labelSingular: string;
+  labelPlural: string;
+  description?: string | null;
+  icon?: string | null;
+};
+
+const classSuffix = 'ObjectMetadata';
+
+export function ObjectMetadata(
+  metadata: ObjectMetadataDecorator,
+): ClassDecorator {
+  return (target) => {
+    const isSystem = Reflect.getMetadata('isSystem', target) || false;
+
+    let objectName = camelCase(target.name);
+
+    if (objectName.endsWith(classSuffix)) {
+      objectName = objectName.slice(0, -classSuffix.length);
+    }
+
+    Reflect.defineMetadata(
+      'objectMetadata',
+      {
+        nameSingular: objectName,
+        ...metadata,
+        targetTableName: objectName,
+        isSystem,
+        isCustom: false,
+        isActive: true,
+      },
+      target,
+    );
+  };
+}
+
+export function IsNullable() {
+  return function (target: object, fieldKey: string) {
+    Reflect.defineMetadata('isNullable', true, target, fieldKey);
+  };
+}
+
+export function IsSystem() {
+  return function (target: object, fieldKey?: string) {
+    if (fieldKey) {
+      Reflect.defineMetadata('isSystem', true, target, fieldKey);
+    } else {
+      Reflect.defineMetadata('isSystem', true, target);
+    }
+  };
+}
+
+export function FieldMetadata(
+  metadata: FieldMetadataDecorator,
+): PropertyDecorator {
+  return (target: object, fieldKey: string) => {
+    const existingFieldMetadata =
+      Reflect.getMetadata('fieldMetadata', target.constructor) || {};
+
+    const isNullable =
+      Reflect.getMetadata('isNullable', target, fieldKey) || false;
+
+    const isSystem = Reflect.getMetadata('isSystem', target, fieldKey) || false;
+
+    Reflect.defineMetadata(
+      'fieldMetadata',
+      {
+        ...existingFieldMetadata,
+        [fieldKey]: {
+          name: fieldKey,
+          ...metadata,
+          targetColumnMap: generateTargetColumnMap(
+            metadata.type,
+            false,
+            fieldKey,
+          ),
+          isNullable,
+          isSystem,
+          isCustom: false,
+          isActive: true,
+        },
+      },
+      target.constructor,
+    );
+  };
+}

server/src/workspace/workspace-manager/standard-objects/api-key.object-metadata.ts

@@ -0,0 +1,44 @@
+import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
+import {
+  FieldMetadata,
+  IsNullable,
+  IsSystem,
+  ObjectMetadata,
+} from 'src/workspace/workspace-manager/decorators/metadata.decorator';
+import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata';
+
+@ObjectMetadata({
+  namePlural: 'apiKeys',
+  labelSingular: 'Api Key',
+  labelPlural: 'Api Keys',
+  description: 'A api key',
+  icon: 'IconRobot',
+})
+@IsSystem()
+export class ApiKeyObjectMetadata extends BaseObjectMetadata {
+  @FieldMetadata({
+    type: FieldMetadataType.TEXT,
+    label: 'Name',
+    description: 'ApiKey name',
+    icon: 'IconLink',
+    defaultValue: { value: '' },
+  })
+  name: string;
+
+  @FieldMetadata({
+    type: FieldMetadataType.DATE_TIME,
+    label: 'Expiration date',
+    description: 'ApiKey expiration date',
+    icon: 'IconCalendar',
+  })
+  expiresAt: Date;
+
+  @FieldMetadata({
+    type: FieldMetadataType.DATE_TIME,
+    label: 'Revocation date',
+    description: 'ApiKey revocation date',
+    icon: 'IconCalendar',
+  })
+  @IsNullable()
+  revokedAt?: Date;
+}

server/src/workspace/workspace-manager/standard-objects/base.object-metadata.ts

@@ -0,0 +1,37 @@
+import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
+import {
+  FieldMetadata,
+  IsSystem,
+} from 'src/workspace/workspace-manager/decorators/metadata.decorator';
+
+export abstract class BaseObjectMetadata {
+  @FieldMetadata({
+    type: FieldMetadataType.UUID,
+    label: 'Id',
+    icon: null,
+    description: null,
+    defaultValue: { type: 'uuid' },
+  })
+  @IsSystem()
+  id: string;
+
+  @FieldMetadata({
+    type: FieldMetadataType.DATE_TIME,
+    label: 'Creation date',
+    description: null,
+    icon: 'IconCalendar',
+    defaultValue: { type: 'now' },
+  })
+  @IsSystem()
+  createdAt: Date;
+
+  @FieldMetadata({
+    type: FieldMetadataType.DATE_TIME,
+    label: 'Update date',
+    description: null,
+    icon: 'IconCalendar',
+    defaultValue: { type: 'now' },
+  })
+  @IsSystem()
+  updatedAt: Date;
+}

server/src/workspace/workspace-manager/standard-objects/standard-object-metadata.ts

@@ -1,23 +1,22 @@
-import activityTargetMetadata from 'src/workspace/workspace-manager/standard-objects/activity-target';
-import activityMetadata from 'src/workspace/workspace-manager/standard-objects/activity';
 import apiKeyMetadata from 'src/workspace/workspace-manager/standard-objects/api-key';
-import attachmentMetadata from 'src/workspace/workspace-manager/standard-objects/attachment';
-import commentMetadata from 'src/workspace/workspace-manager/standard-objects/comment';
-import favoriteMetadata from 'src/workspace/workspace-manager/standard-objects/favorite';
-import opportunityMetadata from 'src/workspace/workspace-manager/standard-objects/opportunity';
-import personMetadata from 'src/workspace/workspace-manager/standard-objects/person';
-import viewMetadata from 'src/workspace/workspace-manager/standard-objects/view';
-import viewFieldMetadata from 'src/workspace/workspace-manager/standard-objects/view-field';
-import viewFilterMetadata from 'src/workspace/workspace-manager/standard-objects/view-filter';
-import viewSortMetadata from 'src/workspace/workspace-manager/standard-objects/view-sort';
-import webhookMetadata from 'src/workspace/workspace-manager/standard-objects/webhook';
-import pipelineStepMetadata from 'src/workspace/workspace-manager/standard-objects/pipeline-step';
-import companyMetadata from 'src/workspace/workspace-manager/standard-objects/company';
-import workspaceMemberMetadata from 'src/workspace/workspace-manager/standard-objects/workspace-member';
 import {
   FieldMetadataEntity,
   FieldMetadataType,
 } from 'src/metadata/field-metadata/field-metadata.entity';
+import activityMetadata from 'src/workspace/workspace-manager/standard-objects/activity';
+import activityTargetMetadata from 'src/workspace/workspace-manager/standard-objects/activity-target';
+import attachmentMetadata from 'src/workspace/workspace-manager/standard-objects/attachment';
+import commentMetadata from 'src/workspace/workspace-manager/standard-objects/comment';
+import companyMetadata from 'src/workspace/workspace-manager/standard-objects/company';
+import favoriteMetadata from 'src/workspace/workspace-manager/standard-objects/favorite';
+import opportunityMetadata from 'src/workspace/workspace-manager/standard-objects/opportunity';
+import personMetadata from 'src/workspace/workspace-manager/standard-objects/person';
+import pipelineStepMetadata from 'src/workspace/workspace-manager/standard-objects/pipeline-step';
+import viewMetadata from 'src/workspace/workspace-manager/standard-objects/view';
+import viewFieldMetadata from 'src/workspace/workspace-manager/standard-objects/view-field';
+import viewFilterMetadata from 'src/workspace/workspace-manager/standard-objects/view-filter';
+import viewSortMetadata from 'src/workspace/workspace-manager/standard-objects/view-sort';
+import workspaceMemberMetadata from 'src/workspace/workspace-manager/standard-objects/workspace-member';
 
 export const standardObjectsMetadata = {
   activityTarget: activityTargetMetadata,
@@ -34,7 +33,6 @@ export const standardObjectsMetadata = {
   viewFilter: viewFilterMetadata,
   viewSort: viewSortMetadata,
   view: viewMetadata,
-  webhook: webhookMetadata,
   workspaceMember: workspaceMemberMetadata,
 };
 

server/src/workspace/workspace-manager/standard-objects/view-sort.object-metadata.ts

@@ -0,0 +1,36 @@
+import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
+import {
+  ObjectMetadata,
+  FieldMetadata,
+  IsNullable,
+  IsSystem,
+} from 'src/workspace/workspace-manager/decorators/metadata.decorator';
+import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata';
+
+@ObjectMetadata({
+  namePlural: 'viewSorts',
+  labelSingular: 'View Sort',
+  labelPlural: 'View Sorts',
+  description: '(System) View Sorts',
+  icon: 'IconArrowsSort',
+})
+@IsSystem()
+export class ViewSortObjectMetadata extends BaseObjectMetadata {
+  @FieldMetadata({
+    type: FieldMetadataType.UUID,
+    label: 'Field Metadata Id',
+    description: 'View Sort target field',
+    icon: null,
+  })
+  fieldMetadataId: string;
+
+  // TODO: We could create a relation decorator but let's keep it simple for now.
+  @FieldMetadata({
+    type: FieldMetadataType.RELATION,
+    label: 'View',
+    description: 'View Sort related view',
+    icon: 'IconLayoutCollage',
+  })
+  @IsNullable()
+  view?: object;
+}

server/src/workspace/workspace-manager/standard-objects/webook.object-metadata.ts

@@ -0,0 +1,35 @@
+import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
+import {
+  FieldMetadata,
+  IsSystem,
+  ObjectMetadata,
+} from 'src/workspace/workspace-manager/decorators/metadata.decorator';
+import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata';
+
+@ObjectMetadata({
+  namePlural: 'webhooks',
+  labelSingular: 'Webhook',
+  labelPlural: 'Webhooks',
+  description: 'A webhook',
+  icon: 'IconRobot',
+})
+@IsSystem()
+export class WebhookObjectMetadata extends BaseObjectMetadata {
+  @FieldMetadata({
+    type: FieldMetadataType.TEXT,
+    label: 'Target Url',
+    description: 'Webhook target url',
+    icon: 'IconLink',
+    defaultValue: { value: '' },
+  })
+  targetUrl: string;
+
+  @FieldMetadata({
+    type: FieldMetadataType.TEXT,
+    label: 'Operation',
+    description: 'Webhook operation',
+    icon: 'IconCheckbox',
+    defaultValue: { value: '' },
+  })
+  operation: string;
+}

server/src/workspace/workspace-manager/utils/metadata.parser.ts

@@ -0,0 +1,41 @@
+import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity';
+import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata';
+
+export class MetadataParser {
+  static parseMetadata(
+    metadata: typeof BaseObjectMetadata,
+    workspaceId: string,
+    dataSourceId: string,
+  ) {
+    const objectMetadata = Reflect.getMetadata('objectMetadata', metadata);
+    const fieldMetadata = Reflect.getMetadata('fieldMetadata', metadata);
+
+    if (objectMetadata) {
+      const fields = Object.values(fieldMetadata);
+      return {
+        ...objectMetadata,
+        workspaceId,
+        dataSourceId,
+        fields: fields.map((field: FieldMetadataEntity) => ({
+          ...field,
+          workspaceId,
+          isSystem: objectMetadata.isSystem || field.isSystem,
+          defaultValue: field.defaultValue || null, // TODO: use default default value based on field type
+          options: field.options || null,
+        })),
+      };
+    }
+
+    return undefined;
+  }
+
+  static parseAllMetadata(
+    metadata: (typeof BaseObjectMetadata)[],
+    workspaceId: string,
+    dataSourceId: string,
+  ) {
+    return metadata.map((_metadata) =>
+      MetadataParser.parseMetadata(_metadata, workspaceId, dataSourceId),
+    );
+  }
+}

server/src/workspace/workspace-manager/utils/sync-metadata.util.spec.ts

@@ -0,0 +1,101 @@
+import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
+import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
+
+import {
+  filterIgnoredProperties,
+  mapObjectMetadataByUniqueIdentifier,
+} from './sync-metadata.util';
+
+describe('filterIgnoredProperties', () => {
+  it('should filter out properties based on the ignore list', () => {
+    const obj = {
+      name: 'John',
+      age: 30,
+      email: 'john@example.com',
+      address: '123 Main St',
+    };
+    const propertiesToIgnore = ['age', 'address'];
+
+    const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore);
+
+    expect(filteredObj).toEqual({
+      name: 'John',
+      email: 'john@example.com',
+    });
+  });
+
+  it('should return the original object if ignore list is empty', () => {
+    const obj = {
+      name: 'John',
+      age: 30,
+      email: 'john@example.com',
+      address: '123 Main St',
+    };
+    const propertiesToIgnore: string[] = [];
+
+    const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore);
+
+    expect(filteredObj).toEqual(obj);
+  });
+
+  it('should return an empty object if the original object is empty', () => {
+    const obj = {};
+    const propertiesToIgnore = ['age', 'address'];
+
+    const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore);
+
+    expect(filteredObj).toEqual({});
+  });
+});
+
+describe('mapObjectMetadataByUniqueIdentifier', () => {
+  it('should convert an array of ObjectMetadataEntity objects into a map', () => {
+    const arr: DeepPartial<ObjectMetadataEntity>[] = [
+      {
+        nameSingular: 'user',
+        fields: [
+          { name: 'id', type: FieldMetadataType.UUID },
+          { name: 'name', type: FieldMetadataType.TEXT },
+        ],
+      },
+      {
+        nameSingular: 'product',
+        fields: [
+          { name: 'id', type: FieldMetadataType.UUID },
+          { name: 'name', type: FieldMetadataType.TEXT },
+          { name: 'price', type: FieldMetadataType.UUID },
+        ],
+      },
+    ];
+
+    const mappedObject = mapObjectMetadataByUniqueIdentifier(
+      arr as ObjectMetadataEntity[],
+    );
+
+    expect(mappedObject).toEqual({
+      user: {
+        nameSingular: 'user',
+        fields: {
+          id: { name: 'id', type: FieldMetadataType.UUID },
+          name: { name: 'name', type: FieldMetadataType.TEXT },
+        },
+      },
+      product: {
+        nameSingular: 'product',
+        fields: {
+          id: { name: 'id', type: FieldMetadataType.UUID },
+          name: { name: 'name', type: FieldMetadataType.TEXT },
+          price: { name: 'price', type: FieldMetadataType.UUID },
+        },
+      },
+    });
+  });
+
+  it('should return an empty map if the input array is empty', () => {
+    const arr: ObjectMetadataEntity[] = [];
+
+    const mappedObject = mapObjectMetadataByUniqueIdentifier(arr);
+
+    expect(mappedObject).toEqual({});
+  });
+});

server/src/workspace/workspace-manager/utils/sync-metadata.util.ts

@@ -0,0 +1,41 @@
+import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
+
+/**
+ * This utility function filters out properties from an object based on a list of properties to ignore.
+ * It returns a new object with only the properties that are not in the ignore list.
+ *
+ * @param obj - The object to filter.
+ * @param propertiesToIgnore - An array of property names to ignore.
+ * @returns A new object with filtered properties.
+ */
+export const filterIgnoredProperties = (
+  obj: any,
+  propertiesToIgnore: string[],
+) => {
+  return Object.fromEntries(
+    Object.entries(obj).filter(([key]) => !propertiesToIgnore.includes(key)),
+  );
+};
+
+/**
+ * This utility function converts an array of ObjectMetadataEntity objects into a map,
+ * where the keys are the nameSingular properties of the objects.
+ * Each object in the map contains the original object metadata and its fields as a nested map.
+ *
+ * @param arr - The array of ObjectMetadataEntity objects to convert.
+ * @returns A map of object metadata, with nameSingular as the key and the object as the value.
+ */
+export const mapObjectMetadataByUniqueIdentifier = (
+  arr: ObjectMetadataEntity[],
+) => {
+  return arr.reduce((acc, curr) => {
+    acc[curr.nameSingular] = {
+      ...curr,
+      fields: curr.fields.reduce((acc, curr) => {
+        acc[curr.name] = curr;
+        return acc;
+      }, {}),
+    };
+    return acc;
+  }, {});
+};

server/src/workspace/workspace-manager/workspace-manager.module.ts

@@ -1,12 +1,14 @@
 import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
 
 import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
-import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module';
 import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
 import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module';
 import { WorkspaceMigrationRunnerModule } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.module';
 import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
 import { RelationMetadataModule } from 'src/metadata/relation-metadata/relation-metadata.module';
+import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity';
+import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
 
 import { WorkspaceManagerService } from './workspace-manager.service';
 
@@ -16,9 +18,12 @@ import { WorkspaceManagerService } from './workspace-manager.service';
     WorkspaceMigrationModule,
     WorkspaceMigrationRunnerModule,
     ObjectMetadataModule,
-    FieldMetadataModule,
     DataSourceModule,
     RelationMetadataModule,
+    TypeOrmModule.forFeature(
+      [FieldMetadataEntity, ObjectMetadataEntity],
+      'metadata',
+    ),
   ],
   exports: [WorkspaceManagerService],
   providers: [WorkspaceManagerService],

server/src/workspace/workspace-manager/workspace-manager.service.ts

@@ -1,7 +1,10 @@
 import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Repository } from 'typeorm';
+import diff from 'microdiff';
 
 import { DataSourceService } from 'src/metadata/data-source/data-source.service';
-import { FieldMetadataService } from 'src/metadata/field-metadata/field-metadata.service';
 import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metadata.service';
 import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service';
 import { WorkspaceMigrationService } from 'src/metadata/workspace-migration/workspace-migration.service';
@@ -16,6 +19,14 @@ import {
   FieldMetadataEntity,
   FieldMetadataType,
 } from 'src/metadata/field-metadata/field-metadata.entity';
+import { MetadataParser } from 'src/workspace/workspace-manager/utils/metadata.parser';
+import { WebhookObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/webook.object-metadata';
+import { ApiKeyObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/api-key.object-metadata';
+import { ViewSortObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/view-sort.object-metadata';
+import {
+  filterIgnoredProperties,
+  mapObjectMetadataByUniqueIdentifier,
+} from 'src/workspace/workspace-manager/utils/sync-metadata.util';
 
 import {
   basicFieldsMetadata,
@@ -29,9 +40,13 @@ export class WorkspaceManagerService {
     private readonly workspaceMigrationService: WorkspaceMigrationService,
     private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
     private readonly objectMetadataService: ObjectMetadataService,
-    private readonly fieldMetadataService: FieldMetadataService,
     private readonly dataSourceService: DataSourceService,
     private readonly relationMetadataService: RelationMetadataService,
+
+    @InjectRepository(ObjectMetadataEntity, 'metadata')
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    @InjectRepository(FieldMetadataEntity, 'metadata')
+    private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
   ) {}
 
   /**
@@ -239,23 +254,167 @@ export class WorkspaceManagerService {
 
   /**
    *
-   * Reset all standard objects and fields metadata for a given workspace
+   * Sync all standard objects and fields metadata for a given workspace and data source
+   * This will update the metadata if it has changed and generate migrations based on the diff.
    *
    * @param dataSourceId
    * @param workspaceId
    */
-  public async resetStandardObjectsAndFieldsMetadata(
+  public async syncStandardObjectsAndFieldsMetadata(
     dataSourceId: string,
     workspaceId: string,
   ) {
-    await this.objectMetadataService.deleteMany({
+    const standardObjects = MetadataParser.parseAllMetadata(
-      workspaceId: { eq: workspaceId },
+      [WebhookObjectMetadata, ApiKeyObjectMetadata, ViewSortObjectMetadata],
+      workspaceId,
+      dataSourceId,
+    );
+    const objectsInDB = await this.objectMetadataRepository.find({
+      where: { workspaceId, dataSourceId, isCustom: false },
+      relations: ['fields'],
     });
 
-    await this.createStandardObjectsAndFieldsMetadata(
+    const objectsInDBByName = mapObjectMetadataByUniqueIdentifier(objectsInDB);
-      dataSourceId,
+    const standardObjectsByName =
-      workspaceId,
+      mapObjectMetadataByUniqueIdentifier(standardObjects);
+
+    const objectsToCreate: ObjectMetadataEntity[] = [];
+    const objectsToDelete = objectsInDB.filter(
+      (objectInDB) => !standardObjectsByName[objectInDB.nameSingular],
     );
+    const objectsToUpdate: Record<string, ObjectMetadataEntity> = {};
+
+    const fieldsToCreate: FieldMetadataEntity[] = [];
+    const fieldsToDelete: FieldMetadataEntity[] = [];
+    const fieldsToUpdate: Record<string, FieldMetadataEntity> = {};
+
+    for (const standardObjectName in standardObjectsByName) {
+      const standardObject = standardObjectsByName[standardObjectName];
+      const objectInDB = objectsInDBByName[standardObjectName];
+
+      if (!objectInDB) {
+        objectsToCreate.push(standardObject);
+        continue;
+      }
+
+      // Deconstruct fields and compare objects and fields independently
+      const { fields: objectInDBFields, ...objectInDBWithoutFields } =
+        objectInDB;
+      const { fields: standardObjectFields, ...standardObjectWithoutFields } =
+        standardObject;
+
+      const objectPropertiesToIgnore = [
+        'id',
+        'createdAt',
+        'updatedAt',
+        'labelIdentifierFieldMetadataId',
+        'imageIdentifierFieldMetadataId',
+      ];
+      const objectDiffWithoutIgnoredProperties = filterIgnoredProperties(
+        objectInDBWithoutFields,
+        objectPropertiesToIgnore,
+      );
+
+      const fieldPropertiesToIgnore = [
+        'id',
+        'createdAt',
+        'updatedAt',
+        'objectMetadataId',
+      ];
+      const objectInDBFieldsWithoutDefaultFields = Object.fromEntries(
+        Object.entries(objectInDBFields).map(([key, value]) => {
+          if (value === null || typeof value !== 'object') {
+            return [key, value];
+          }
+          return [key, filterIgnoredProperties(value, fieldPropertiesToIgnore)];
+        }),
+      );
+
+      // Compare objects
+      const objectDiff = diff(
+        objectDiffWithoutIgnoredProperties,
+        standardObjectWithoutFields,
+      );
+
+      // Compare fields
+      const fieldsDiff = diff(
+        objectInDBFieldsWithoutDefaultFields,
+        standardObjectFields,
+      );
+
+      for (const diff of objectDiff) {
+        // We only handle CHANGE here as REMOVE and CREATE are handled earlier.
+        if (diff.type === 'CHANGE') {
+          const property = diff.path[0];
+          objectsToUpdate[objectInDB.id] = {
+            ...objectsToUpdate[objectInDB.id],
+            [property]: diff.value,
+          };
+        }
+      }
+
+      for (const diff of fieldsDiff) {
+        if (diff.type === 'CREATE') {
+          const fieldName = diff.path[0];
+          const fieldMetadata = standardObjectFields[fieldName];
+          fieldsToCreate.push(fieldMetadata);
+        }
+        if (diff.type === 'CHANGE') {
+          const fieldName = diff.path[0];
+          const property = diff.path[diff.path.length - 1];
+          const fieldMetadata = objectInDBFields[fieldName];
+          fieldsToUpdate[fieldMetadata.id] = {
+            ...fieldsToUpdate[fieldMetadata.id],
+            [property]: diff.value,
+          };
+        }
+        if (diff.type === 'REMOVE') {
+          const fieldName = diff.path[0];
+          const fieldMetadata = objectInDBFields[fieldName];
+          fieldsToDelete.push(fieldMetadata);
+        }
+      }
+      // console.log(standardObjectName + ':objectDiff', objectDiff);
+      // console.log(standardObjectName + ':fieldsDiff', fieldsDiff);
+    }
+
+    // TODO: Sync relationMetadata
+    // NOTE: Relations are handled like any field during the diff, so we ignore the relationMetadata table
+    // during the diff as it depends on the 2 fieldMetadata that we will compare here.
+    // However we need to make sure the relationMetadata table is in sync with the fieldMetadata table.
+
+    // TODO: Use transactions
+    // CREATE OBJECTS
+    try {
+      await this.objectMetadataRepository.save(objectsToCreate);
+      // UPDATE OBJECTS, this is not optimal as we are running n queries here.
+      for (const [key, value] of Object.entries(objectsToUpdate)) {
+        await this.objectMetadataRepository.update(key, value);
+      }
+      // DELETE OBJECTS
+      if (objectsToDelete.length > 0) {
+        await this.objectMetadataRepository.delete(
+          objectsToDelete.map((object) => object.id),
+        );
+      }
+
+      // CREATE FIELDS
+      await this.fieldMetadataRepository.save(fieldsToCreate);
+      // UPDATE FIELDS
+      for (const [key, value] of Object.entries(fieldsToUpdate)) {
+        await this.fieldMetadataRepository.update(key, value);
+      }
+      // DELETE FIELDS
+      if (fieldsToDelete.length > 0) {
+        await this.fieldMetadataRepository.delete(
+          fieldsToDelete.map((field) => field.id),
+        );
+      }
+    } catch (e) {
+      console.error('Sync of standard objects failed with:', e);
+    }
+
+    // TODO: Create migrations based on diff from above.
   }
 
   /**

server/yarn.lock

@@ -6951,6 +6951,11 @@ methods@^1.1.2, methods@~1.1.2:
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==
 
+microdiff@^1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/microdiff/-/microdiff-1.3.2.tgz#b4fec53aca97371d5409a354913a65be2daec11d"
+  integrity sha512-pKy60S2febliZIbwdfEQKTtL5bLNxOyiRRmD400gueYl9XcHyNGxzHSlJWn9IMHwYXT0yohPYL08+bGozVk8cQ==
+
 micromatch@^4.0.0, micromatch@^4.0.2, micromatch@^4.0.4:
   version "4.0.5"
   resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6"