Standard migration command (#2236)

* Add Standard Object migration commands

* rebase

* add sync-tenant-metadata command

* fix naming

* renaming command class names

* remove field deletion and use object cascade instead

---------

Co-authored-by: Charles Bochet <charlesBochet@users.noreply.github.com>

server/src/command.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+
+import { AppModule } from './app.module';
+
+import { MetadataCommandModule } from './metadata/commands/metadata-command.module';
+
+@Module({
+  imports: [AppModule, MetadataCommandModule],
+})
+export class CommandModule {}

server/src/command.ts

@@ -0,0 +1,9 @@
+import { CommandFactory } from 'nest-commander';
+
+import { CommandModule } from './command.module';
+
+async function bootstrap() {
+  // TODO: inject our own logger service to handle the output (Sentry, etc.)
+  await CommandFactory.run(CommandModule, ['warn', 'error']);
+}
+bootstrap();

server/src/metadata/commands/metadata-command.module.ts

@@ -0,0 +1,24 @@
+import { Module } from '@nestjs/common';
+
+import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module';
+import { MigrationRunnerModule } from 'src/metadata/migration-runner/migration-runner.module';
+import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
+import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module';
+import { TenantInitialisationModule } from 'src/metadata/tenant-initialisation/tenant-initialisation.module';
+import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
+
+import { SyncTenantMetadataCommand } from './sync-tenant-metadata.command';
+import { RunTenantMigrationsCommand } from './run-tenant-migrations.command';
+
+@Module({
+  imports: [
+    TenantMigrationModule,
+    MigrationRunnerModule,
+    ObjectMetadataModule,
+    FieldMetadataModule,
+    DataSourceMetadataModule,
+    TenantInitialisationModule,
+  ],
+  providers: [RunTenantMigrationsCommand, SyncTenantMetadataCommand],
+})
+export class MetadataCommandModule {}

server/src/metadata/commands/run-tenant-migrations.command.ts

@@ -0,0 +1,45 @@
+import { Command, CommandRunner, Option } from 'nest-commander';
+
+import { TenantMigrationService } from 'src/metadata/tenant-migration/tenant-migration.service';
+import { MigrationRunnerService } from 'src/metadata/migration-runner/migration-runner.service';
+
+// TODO: implement dry-run
+interface RunTenantMigrationsOptions {
+  workspaceId: string;
+}
+
+@Command({
+  name: 'tenant:migrate',
+  description: 'Run tenant migrations',
+})
+export class RunTenantMigrationsCommand extends CommandRunner {
+  constructor(
+    private readonly tenantMigrationService: TenantMigrationService,
+    private readonly migrationRunnerService: MigrationRunnerService,
+  ) {
+    super();
+  }
+
+  async run(
+    _passedParam: string[],
+    options: RunTenantMigrationsOptions,
+  ): Promise<void> {
+    // TODO: run in a dedicated job + run queries in a transaction.
+    await this.tenantMigrationService.insertStandardMigrations(
+      options.workspaceId,
+    );
+    await this.migrationRunnerService.executeMigrationFromPendingMigrations(
+      options.workspaceId,
+    );
+  }
+
+  // TODO: workspaceId should be optional and we should run migrations for all workspaces
+  @Option({
+    flags: '-w, --workspace-id [workspace_id]',
+    description: 'workspace id',
+    required: true,
+  })
+  parseWorkspaceId(value: string): string {
+    return value;
+  }
+}

server/src/metadata/commands/sync-tenant-metadata.command.ts

@@ -0,0 +1,57 @@
+import { Command, CommandRunner, Option } from 'nest-commander';
+
+import { ObjectMetadataService } from 'src/metadata/object-metadata/services/object-metadata.service';
+import { FieldMetadataService } from 'src/metadata/field-metadata/services/field-metadata.service';
+import { TenantInitialisationService } from 'src/metadata/tenant-initialisation/tenant-initialisation.service';
+import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
+
+// TODO: implement dry-run
+interface RunTenantMigrationsOptions {
+  workspaceId: string;
+}
+
+@Command({
+  name: 'tenant:sync-metadata',
+  description: 'Sync metadata',
+})
+export class SyncTenantMetadataCommand extends CommandRunner {
+  constructor(
+    private readonly objectMetadataService: ObjectMetadataService,
+    private readonly fieldMetadataService: FieldMetadataService,
+    private readonly dataSourceMetadataService: DataSourceMetadataService,
+    private readonly tenantInitialisationService: TenantInitialisationService,
+  ) {
+    super();
+  }
+
+  async run(
+    _passedParam: string[],
+    options: RunTenantMigrationsOptions,
+  ): Promise<void> {
+    // TODO: run in a dedicated job + run queries in a transaction.
+    const dataSourceMetadata =
+      await this.dataSourceMetadataService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
+        options.workspaceId,
+      );
+
+    // TODO: This solution could be improved, using a diff for example, we should not have to delete all metadata and recreate them.
+    await this.objectMetadataService.deleteMany({
+      workspaceId: { eq: options.workspaceId },
+    });
+
+    // TODO: this should not be the responsibility of tenantInitialisationService.
+    await this.tenantInitialisationService.createObjectsAndFieldsMetadata(
+      dataSourceMetadata.id,
+      options.workspaceId,
+    );
+  }
+
+  @Option({
+    flags: '-w, --workspace-id [workspace_id]',
+    description: 'workspace id',
+    required: true,
+  })
+  parseWorkspaceId(value: string): string {
+    return value;
+  }
+}

server/src/metadata/data-source/data-source.service.ts

@@ -132,5 +132,10 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
   async onModuleDestroy() {
     // Destroy main data source "default" schema
     await this.mainDataSource.destroy();
+
+    // Destroy all workspace data sources
+    for (const [, dataSource] of this.dataSources) {
+      await dataSource.destroy();
+    }
   }
 }

server/src/metadata/migration-runner/migration-runner.service.ts

@@ -35,6 +35,10 @@ export class MigrationRunnerService {
     const pendingMigrations =
       await this.tenantMigrationService.getPendingMigrations(workspaceId);
 
+    if (pendingMigrations.length === 0) {
+      return [];
+    }
+
     const flattenedPendingMigrations: TenantMigrationTableAction[] =
       pendingMigrations.reduce((acc, pendingMigration) => {
         return [...acc, ...pendingMigration.migrations];

server/src/metadata/tenant-initialisation/standard-objects/people/people.metadata.json

@@ -0,0 +1,113 @@
+{
+  "nameSingular": "personV2",
+  "namePlural": "peopleV2",
+  "labelSingular": "Person",
+  "labelPlural": "People",
+  "targetTableName": "person",
+  "description": "A person",
+  "icon": "people",
+  "fields": [
+    {
+      "type": "text",
+      "name": "firstName",
+      "label": "First Name",
+      "targetColumnMap": {
+        "value": "firstName"
+      },
+      "description": "First Name of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "lastName",
+      "label": "Last Name",
+      "targetColumnMap": {
+        "value": "lastName"
+      },
+      "description": "Last Name of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "email",
+      "label": "Email",
+      "targetColumnMap": {
+        "value": "email"
+      },
+      "description": "Email of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "phone",
+      "name": "phone",
+      "label": "Phone",
+      "targetColumnMap": {
+        "value": "phone"
+      },
+      "description": "phone of the company",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "city",
+      "label": "City",
+      "targetColumnMap": {
+        "value": "city"
+      },
+      "description": "City of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "jobTitle",
+      "label": "Job Title",
+      "targetColumnMap": {
+        "value": "jobTitle"
+      },
+      "description": "Job title of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "url",
+      "name": "linkedinUrl",
+      "label": "Linkedin URL",
+      "targetColumnMap": {
+        "text": "Linkedin URL",
+        "link": "linkedinUrl"
+      },
+      "description": "Linkedin URL of the person",
+      "icon": "url",
+      "isNullable": true
+    },
+    {
+      "type": "url",
+      "name": "xUrl",
+      "label": "X URL",
+      "targetColumnMap": {
+        "text": "X URL",
+        "link": "xUrl"
+      },
+      "description": "X URL of the person",
+      "icon": "url",
+      "isNullable": true
+    },
+    {
+      "type": "url",
+      "name": "avatarUrl",
+      "label": "Avatar URL",
+      "targetColumnMap": {
+        "text": "Avatar URL",
+        "link": "avatarUrl"
+      },
+      "description": "Avatar URL of the person",
+      "icon": "url",
+      "isNullable": true
+    }
+  ]
+}

server/src/metadata/tenant-initialisation/standard-objects/standard-object-metadata.ts

@@ -1,5 +1,7 @@
-import companyObject from './companies.metadata.json';
+import companyObject from './companies/companies.metadata.json';
+import personObject from './people/people.metadata.json';
 
 export const standardObjectsMetadata = {
   companyV2: companyObject,
+  personV2: personObject,
 };

server/src/metadata/tenant-initialisation/standard-objects/standard-object-seeds.ts

@@ -1,4 +1,4 @@
-import companySeeds from './companies.seeds.json';
+import companySeeds from './companies/companies.seeds.json';
 
 export const standardObjectsSeeds = {
   companyV2: companySeeds,

server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts

@@ -67,7 +67,7 @@ export class TenantInitialisationService {
    * @param dataSourceMetadataId
    * @param workspaceId
    */
-  private async createObjectsAndFieldsMetadata(
+  public async createObjectsAndFieldsMetadata(
     dataSourceMetadataId: string,
     workspaceId: string,
   ) {

server/src/metadata/tenant-migration/migrations/1697618010-addPeopleTable.ts

@@ -0,0 +1,59 @@
+import { TenantMigrationTableAction } from 'src/metadata/tenant-migration/tenant-migration.entity';
+
+export const addPeopleTable: TenantMigrationTableAction[] = [
+  {
+    name: 'people',
+    action: 'create',
+  },
+  {
+    name: 'people',
+    action: 'alter',
+    columns: [
+      {
+        name: 'firstName',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'lastName',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'email',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'phone',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'city',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'jobTitle',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'linkedinUrl',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'xUrl',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'avatarUrl',
+        type: 'varchar',
+        action: 'create',
+      },
+    ],
+  },
+];

server/src/metadata/tenant-migration/standard-migrations.ts

@@ -1,6 +1,8 @@
 import { addCompanyTable } from './migrations/1697618009-addCompanyTable';
+import { addPeopleTable } from './migrations/1697618010-addPeopleTable';
 
 // TODO: read the folder and return all migrations
 export const standardMigrations = {
   '1697618009-addCompanyTable': addCompanyTable,
+  '1697618010-addPeopleTable': addPeopleTable,
 };

server/src/metadata/tenant-migration/tenant-migration.service.ts

@@ -21,7 +21,7 @@ export class TenantMigrationService {
    *
    * @param workspaceId
    */
-  public async insertStandardMigrations(workspaceId: string) {
+  public async insertStandardMigrations(workspaceId: string): Promise<void> {
     // TODO: we actually don't need to fetch all of them, to improve later so it scales well.
     const insertedStandardMigrations =
       await this.tenantMigrationRepository.find({
@@ -34,20 +34,21 @@ export class TenantMigrationService {
         return acc;
       }, {});
 
-    const standardMigrationsList = standardMigrations;
-
     const standardMigrationsListThatNeedToBeInserted = Object.entries(
-      standardMigrationsList,
+      standardMigrations,
     )
       .filter(([name]) => !insertedStandardMigrationsMapByName[name])
       .map(([name, migrations]) => ({ name, migrations }));
 
-    await this.tenantMigrationRepository.save(
+    const standardMigrationsThatNeedToBeInserted =
       standardMigrationsListThatNeedToBeInserted.map((migration) => ({
         ...migration,
         workspaceId,
         isCustom: false,
-      })),
+      }));
+
+    await this.tenantMigrationRepository.save(
+      standardMigrationsThatNeedToBeInserted,
     );
   }
 
@@ -60,7 +61,7 @@ export class TenantMigrationService {
   public async getPendingMigrations(
     workspaceId: string,
   ): Promise<TenantMigration[]> {
-    return this.tenantMigrationRepository.find({
+    return await this.tenantMigrationRepository.find({
       order: { createdAt: 'ASC' },
       where: {
         appliedAt: IsNull(),