feat: workspace sync (#3505)

* feat: wip workspace sync * feat: wip lot of debugging * feat: refactor and fix sync * fix: clean fix: clean * feat: add simple comparator tests * fix: remove debug * feat: wip drop table * fix: main merge * fix: some issues, and prepare storage system to handle complex deletion * feat: wip clean and fix * fix: reflect issue when using array instead of map and clean * fix: test & sync * fix: yarn files * fix: unecesary if-else * fix: if condition not needed * fix: remove debug * fix: replace EQUAL by SKIP * fix: sync metadata relation not applied properly * fix: lint issues * fix: merge issue
2024-01-30 14:40:55 +01:00
parent 3a480f1506
commit 73f6876641
59 changed files with 2103 additions and 927 deletions
--- a/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.service.ts
+++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.service.ts
@ -0,0 +1,82 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { InjectDataSource } from '@nestjs/typeorm';
+
+import { DataSource } from 'typeorm';
+
+import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
+
+import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service';
+import { FeatureFlagFactory } from 'src/workspace/workspace-sync-metadata/factories/feature-flags.factory';
+import { WorkspaceSyncObjectMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service';
+import { WorkspaceSyncRelationMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service';
+import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
+
+@Injectable()
+export class WorkspaceSyncMetadataService {
+  private readonly logger = new Logger(WorkspaceSyncMetadataService.name);
+
+  constructor(
+    @InjectDataSource('metadata')
+    private readonly metadataDataSource: DataSource,
+    private readonly featureFlagFactory: FeatureFlagFactory,
+    private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
+    private readonly workspaceSyncObjectMetadataService: WorkspaceSyncObjectMetadataService,
+    private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService,
+  ) {}
+
+  /**
+   *
+   * Sync all standard objects and fields metadata for a given workspace and data source
+   * This will update the metadata if it has changed and generate migrations based on the diff.
+   *
+   * @param dataSourceId
+   * @param workspaceId
+   */
+  public async syncStandardObjectsAndFieldsMetadata(
+    context: WorkspaceSyncContext,
+  ) {
+    this.logger.log('Syncing standard objects and fields metadata');
+    const queryRunner = this.metadataDataSource.createQueryRunner();
+
+    await queryRunner.connect();
+    await queryRunner.startTransaction();
+
+    const manager = queryRunner.manager;
+
+    try {
+      const storage = new WorkspaceSyncStorage();
+
+      // Retrieve feature flags
+      const workspaceFeatureFlagsMap =
+        await this.featureFlagFactory.create(context);
+
+      this.logger.log('Syncing standard objects and fields metadata');
+
+      await this.workspaceSyncObjectMetadataService.synchronize(
+        context,
+        manager,
+        storage,
+        workspaceFeatureFlagsMap,
+      );
+
+      await this.workspaceSyncRelationMetadataService.synchronize(
+        context,
+        manager,
+        storage,
+        workspaceFeatureFlagsMap,
+      );
+
+      await queryRunner.commitTransaction();
+
+      // Execute migrations
+      await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
+        context.workspaceId,
+      );
+    } catch (error) {
+      console.error('Sync of standard objects failed with:', error);
+      await queryRunner.rollbackTransaction();
+    } finally {
+      await queryRunner.release();
+    }
+  }
+}