Migrate to a monorepo structure (#2909)

packages/twenty-server/src/workspace/workspace-migration-runner/commands/run-workspace-migrations.command.ts

@@ -0,0 +1,45 @@
+import { Command, CommandRunner, Option } from 'nest-commander';
+
+import { WorkspaceMigrationService } from 'src/metadata/workspace-migration/workspace-migration.service';
+import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service';
+
+// TODO: implement dry-run
+interface RunWorkspaceMigrationsOptions {
+  workspaceId: string;
+}
+
+@Command({
+  name: 'workspace:migrate',
+  description: 'Run workspace migrations',
+})
+export class RunWorkspaceMigrationsCommand extends CommandRunner {
+  constructor(
+    private readonly workspaceMigrationService: WorkspaceMigrationService,
+    private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
+  ) {
+    super();
+  }
+
+  async run(
+    _passedParam: string[],
+    options: RunWorkspaceMigrationsOptions,
+  ): Promise<void> {
+    // TODO: run in a dedicated job + run queries in a transaction.
+    await this.workspaceMigrationService.insertStandardMigrations(
+      options.workspaceId,
+    );
+    await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
+      options.workspaceId,
+    );
+  }
+
+  // TODO: workspaceId should be optional and we should run migrations for all workspaces
+  @Option({
+    flags: '-w, --workspace-id [workspace_id]',
+    description: 'workspace id',
+    required: true,
+  })
+  parseWorkspaceId(value: string): string {
+    return value;
+  }
+}

packages/twenty-server/src/workspace/workspace-migration-runner/commands/workspace-migration-runner-commands.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+
+import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module';
+import { WorkspaceMigrationRunnerModule } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.module';
+
+import { RunWorkspaceMigrationsCommand } from './run-workspace-migrations.command';
+
+@Module({
+  imports: [WorkspaceMigrationModule, WorkspaceMigrationRunnerModule],
+  providers: [RunWorkspaceMigrationsCommand],
+})
+export class WorkspaceMigrationRunnerCommandsModule {}

packages/twenty-server/src/workspace/workspace-migration-runner/services/workspace-migration-enum.service.ts

@@ -0,0 +1,187 @@
+import { Injectable } from '@nestjs/common';
+
+import { QueryRunner } from 'typeorm';
+
+import { WorkspaceMigrationColumnAlter } from 'src/metadata/workspace-migration/workspace-migration.entity';
+
+@Injectable()
+export class WorkspaceMigrationEnumService {
+  async alterEnum(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    migrationColumn: WorkspaceMigrationColumnAlter,
+  ) {
+    const columnDefinition = migrationColumn.alteredColumnDefinition;
+    const oldEnumTypeName = `${tableName}_${columnDefinition.columnName}_enum`;
+    const newEnumTypeName = `${tableName}_${columnDefinition.columnName}_enum_new`;
+    const enumValues =
+      columnDefinition.enum?.map((enumValue) => {
+        if (typeof enumValue === 'string') {
+          return enumValue;
+        }
+
+        return enumValue.to;
+      }) ?? [];
+
+    if (!columnDefinition.isNullable && !columnDefinition.defaultValue) {
+      columnDefinition.defaultValue = columnDefinition.enum?.[0];
+    }
+
+    // Create new enum type with new values
+    await this.createNewEnumType(
+      newEnumTypeName,
+      queryRunner,
+      schemaName,
+      enumValues,
+    );
+
+    // Temporarily change column type to text
+    await queryRunner.query(`
+      ALTER TABLE "${schemaName}"."${tableName}"
+      ALTER COLUMN "${columnDefinition.columnName}" TYPE TEXT
+    `);
+
+    // Migrate existing values to new values
+    await this.migrateEnumValues(
+      queryRunner,
+      schemaName,
+      tableName,
+      migrationColumn,
+    );
+
+    // Update existing rows to handle missing values
+    await this.handleMissingEnumValues(
+      queryRunner,
+      schemaName,
+      tableName,
+      migrationColumn,
+      enumValues,
+    );
+
+    // Alter column type to new enum
+    await this.updateColumnToNewEnum(
+      queryRunner,
+      schemaName,
+      tableName,
+      columnDefinition.columnName,
+      newEnumTypeName,
+    );
+
+    // Drop old enum type
+    await this.dropOldEnumType(queryRunner, schemaName, oldEnumTypeName);
+
+    // Rename new enum type to old enum type name
+    await this.renameEnumType(
+      queryRunner,
+      schemaName,
+      oldEnumTypeName,
+      newEnumTypeName,
+    );
+  }
+
+  private async createNewEnumType(
+    name: string,
+    queryRunner: QueryRunner,
+    schemaName: string,
+    newValues: string[],
+  ) {
+    const enumValues = newValues
+      .map((value) => `'${value.replace(/'/g, "''")}'`)
+      .join(', ');
+
+    await queryRunner.query(
+      `CREATE TYPE "${schemaName}"."${name}" AS ENUM (${enumValues})`,
+    );
+  }
+
+  private async migrateEnumValues(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    migrationColumn: WorkspaceMigrationColumnAlter,
+  ) {
+    const columnDefinition = migrationColumn.alteredColumnDefinition;
+
+    if (!columnDefinition.enum) {
+      return;
+    }
+
+    for (const enumValue of columnDefinition.enum) {
+      // Skip string values
+      if (typeof enumValue === 'string') {
+        continue;
+      }
+      await queryRunner.query(`
+        UPDATE "${schemaName}"."${tableName}"
+        SET "${columnDefinition.columnName}" = '${enumValue.to}'
+        WHERE "${columnDefinition.columnName}" = '${enumValue.from}'
+      `);
+    }
+  }
+
+  private async handleMissingEnumValues(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    migrationColumn: WorkspaceMigrationColumnAlter,
+    enumValues: string[],
+  ) {
+    const columnDefinition = migrationColumn.alteredColumnDefinition;
+
+    // Set missing values to null or default value
+    let defaultValue = 'NULL';
+
+    if (columnDefinition.defaultValue) {
+      if (Array.isArray(columnDefinition.defaultValue)) {
+        defaultValue = `ARRAY[${columnDefinition.defaultValue
+          .map((e) => `'${e}'`)
+          .join(', ')}]`;
+      } else {
+        defaultValue = `'${columnDefinition.defaultValue}'`;
+      }
+    }
+
+    await queryRunner.query(`
+      UPDATE "${schemaName}"."${tableName}"
+      SET "${columnDefinition.columnName}" = ${defaultValue}
+      WHERE "${columnDefinition.columnName}" NOT IN (${enumValues
+      .map((e) => `'${e}'`)
+      .join(', ')})
+    `);
+  }
+
+  private async updateColumnToNewEnum(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    columnName: string,
+    newEnumTypeName: string,
+  ) {
+    await queryRunner.query(
+      `ALTER TABLE "${schemaName}"."${tableName}" ALTER COLUMN "${columnName}" TYPE "${schemaName}"."${newEnumTypeName}" USING ("${columnName}"::text::"${schemaName}"."${newEnumTypeName}")`,
+    );
+  }
+
+  private async dropOldEnumType(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    oldEnumTypeName: string,
+  ) {
+    await queryRunner.query(
+      `DROP TYPE IF EXISTS "${schemaName}"."${oldEnumTypeName}"`,
+    );
+  }
+
+  private async renameEnumType(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    oldEnumTypeName: string,
+    newEnumTypeName: string,
+  ) {
+    await queryRunner.query(`
+      ALTER TYPE "${schemaName}"."${newEnumTypeName}"
+      RENAME TO "${oldEnumTypeName}"
+    `);
+  }
+}

packages/twenty-server/src/workspace/workspace-migration-runner/utils/custom-table-default-column.util.ts

@@ -0,0 +1,25 @@
+import { TableColumnOptions } from 'typeorm';
+
+export const customTableDefaultColumns: TableColumnOptions[] = [
+  {
+    name: 'id',
+    type: 'uuid',
+    isPrimary: true,
+    default: 'public.uuid_generate_v4()',
+  },
+  {
+    name: 'createdAt',
+    type: 'timestamp',
+    default: 'now()',
+  },
+  {
+    name: 'updatedAt',
+    type: 'timestamp',
+    default: 'now()',
+  },
+  {
+    name: 'deletedAt',
+    type: 'timestamp',
+    isNullable: true,
+  },
+];

packages/twenty-server/src/workspace/workspace-migration-runner/workspace-migration-runner.module.ts

@@ -0,0 +1,19 @@
+import { Module } from '@nestjs/common';
+
+import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module';
+import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
+import { WorkspaceCacheVersionModule } from 'src/metadata/workspace-cache-version/workspace-cache-version.module';
+import { WorkspaceMigrationEnumService } from 'src/workspace/workspace-migration-runner/services/workspace-migration-enum.service';
+
+import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.service';
+
+@Module({
+  imports: [
+    WorkspaceDataSourceModule,
+    WorkspaceMigrationModule,
+    WorkspaceCacheVersionModule,
+  ],
+  providers: [WorkspaceMigrationRunnerService, WorkspaceMigrationEnumService],
+  exports: [WorkspaceMigrationRunnerService],
+})
+export class WorkspaceMigrationRunnerModule {}

packages/twenty-server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts

@@ -0,0 +1,306 @@
+import { Injectable } from '@nestjs/common';
+
+import {
+  QueryRunner,
+  Table,
+  TableColumn,
+  TableForeignKey,
+  TableUnique,
+} from 'typeorm';
+
+import { WorkspaceMigrationService } from 'src/metadata/workspace-migration/workspace-migration.service';
+import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
+import {
+  WorkspaceMigrationTableAction,
+  WorkspaceMigrationColumnAction,
+  WorkspaceMigrationColumnActionType,
+  WorkspaceMigrationColumnCreate,
+  WorkspaceMigrationColumnRelation,
+  WorkspaceMigrationColumnAlter,
+} from 'src/metadata/workspace-migration/workspace-migration.entity';
+import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service';
+import { WorkspaceMigrationEnumService } from 'src/workspace/workspace-migration-runner/services/workspace-migration-enum.service';
+
+import { customTableDefaultColumns } from './utils/custom-table-default-column.util';
+
+@Injectable()
+export class WorkspaceMigrationRunnerService {
+  constructor(
+    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
+    private readonly workspaceMigrationService: WorkspaceMigrationService,
+    private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
+    private readonly workspaceMigrationEnumService: WorkspaceMigrationEnumService,
+  ) {}
+
+  /**
+   * Executes pending migrations for a given workspace
+   *
+   * @param workspaceId string
+   * @returns Promise<WorkspaceMigrationTableAction[]>
+   */
+  public async executeMigrationFromPendingMigrations(
+    workspaceId: string,
+  ): Promise<WorkspaceMigrationTableAction[]> {
+    const workspaceDataSource =
+      await this.workspaceDataSourceService.connectToWorkspaceDataSource(
+        workspaceId,
+      );
+
+    if (!workspaceDataSource) {
+      throw new Error('Workspace data source not found');
+    }
+
+    const pendingMigrations =
+      await this.workspaceMigrationService.getPendingMigrations(workspaceId);
+
+    if (pendingMigrations.length === 0) {
+      return [];
+    }
+
+    const flattenedPendingMigrations: WorkspaceMigrationTableAction[] =
+      pendingMigrations.reduce((acc, pendingMigration) => {
+        return [...acc, ...pendingMigration.migrations];
+      }, []);
+
+    const queryRunner = workspaceDataSource?.createQueryRunner();
+    const schemaName =
+      this.workspaceDataSourceService.getSchemaName(workspaceId);
+
+    // Loop over each migration and create or update the table
+    // TODO: Should be done in a transaction
+    for (const migration of flattenedPendingMigrations) {
+      await this.handleTableChanges(queryRunner, schemaName, migration);
+    }
+
+    // Update appliedAt date for each migration
+    // TODO: Should be done after the migration is successful
+    for (const pendingMigration of pendingMigrations) {
+      await this.workspaceMigrationService.setAppliedAtForMigration(
+        workspaceId,
+        pendingMigration,
+      );
+    }
+
+    await queryRunner.release();
+
+    // Increment workspace cache version
+    await this.workspaceCacheVersionService.incrementVersion(workspaceId);
+
+    return flattenedPendingMigrations;
+  }
+
+  /**
+   * Handles table changes for a given migration
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableMigration WorkspaceMigrationTableChange
+   */
+  private async handleTableChanges(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableMigration: WorkspaceMigrationTableAction,
+  ) {
+    switch (tableMigration.action) {
+      case 'create':
+        await this.createTable(queryRunner, schemaName, tableMigration.name);
+        break;
+      case 'alter':
+        await this.handleColumnChanges(
+          queryRunner,
+          schemaName,
+          tableMigration.name,
+          tableMigration?.columns,
+        );
+        break;
+      default:
+        throw new Error(
+          `Migration table action ${tableMigration.action} not supported`,
+        );
+    }
+  }
+
+  /**
+   * Creates a table for a given schema and table name
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableName string
+   */
+  private async createTable(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+  ) {
+    await queryRunner.createTable(
+      new Table({
+        name: tableName,
+        schema: schemaName,
+        columns: customTableDefaultColumns,
+      }),
+      true,
+    );
+  }
+
+  /**
+   * Handles column changes for a given migration
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableName string
+   * @param columnMigrations WorkspaceMigrationColumnAction[]
+   * @returns
+   */
+  private async handleColumnChanges(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    columnMigrations?: WorkspaceMigrationColumnAction[],
+  ) {
+    if (!columnMigrations || columnMigrations.length === 0) {
+      return;
+    }
+
+    for (const columnMigration of columnMigrations) {
+      switch (columnMigration.action) {
+        case WorkspaceMigrationColumnActionType.CREATE:
+          await this.createColumn(
+            queryRunner,
+            schemaName,
+            tableName,
+            columnMigration,
+          );
+          break;
+        case WorkspaceMigrationColumnActionType.ALTER:
+          await this.alterColumn(
+            queryRunner,
+            schemaName,
+            tableName,
+            columnMigration,
+          );
+          break;
+        case WorkspaceMigrationColumnActionType.RELATION:
+          await this.createForeignKey(
+            queryRunner,
+            schemaName,
+            tableName,
+            columnMigration,
+          );
+          break;
+        default:
+          throw new Error(`Migration column action not supported`);
+      }
+    }
+  }
+
+  /**
+   * Creates a column for a given schema, table name, and column migration
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableName string
+   * @param migrationColumn WorkspaceMigrationColumnAction
+   */
+  private async createColumn(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    migrationColumn: WorkspaceMigrationColumnCreate,
+  ) {
+    const hasColumn = await queryRunner.hasColumn(
+      `${schemaName}.${tableName}`,
+      migrationColumn.columnName,
+    );
+
+    if (hasColumn) {
+      return;
+    }
+
+    await queryRunner.addColumn(
+      `${schemaName}.${tableName}`,
+      new TableColumn({
+        name: migrationColumn.columnName,
+        type: migrationColumn.columnType,
+        default: migrationColumn.defaultValue,
+        enum: migrationColumn.enum?.filter(
+          (value): value is string => typeof value === 'string',
+        ),
+        isArray: migrationColumn.isArray,
+        isNullable: true,
+      }),
+    );
+  }
+
+  private async alterColumn(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    migrationColumn: WorkspaceMigrationColumnAlter,
+  ) {
+    const enumValues = migrationColumn.alteredColumnDefinition.enum;
+
+    // TODO: Maybe we can do something better if we can recreate the old `TableColumn` object
+    if (enumValues) {
+      // This is returning the old enum values to avoid TypeORM droping the enum type
+      await this.workspaceMigrationEnumService.alterEnum(
+        queryRunner,
+        schemaName,
+        tableName,
+        migrationColumn,
+      );
+    }
+
+    await queryRunner.changeColumn(
+      `${schemaName}.${tableName}`,
+      new TableColumn({
+        name: migrationColumn.currentColumnDefinition.columnName,
+        type: migrationColumn.currentColumnDefinition.columnType,
+        default: migrationColumn.currentColumnDefinition.defaultValue,
+        enum: migrationColumn.currentColumnDefinition.enum?.filter(
+          (value): value is string => typeof value === 'string',
+        ),
+        isArray: migrationColumn.currentColumnDefinition.isArray,
+        isNullable: migrationColumn.currentColumnDefinition.isNullable,
+      }),
+      new TableColumn({
+        name: migrationColumn.alteredColumnDefinition.columnName,
+        type: migrationColumn.alteredColumnDefinition.columnType,
+        default: migrationColumn.alteredColumnDefinition.defaultValue,
+        enum: migrationColumn.currentColumnDefinition.enum?.filter(
+          (value): value is string => typeof value === 'string',
+        ),
+        isArray: migrationColumn.alteredColumnDefinition.isArray,
+        isNullable: migrationColumn.alteredColumnDefinition.isNullable,
+      }),
+    );
+    // }
+  }
+
+  private async createForeignKey(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    migrationColumn: WorkspaceMigrationColumnRelation,
+  ) {
+    await queryRunner.createForeignKey(
+      `${schemaName}.${tableName}`,
+      new TableForeignKey({
+        columnNames: [migrationColumn.columnName],
+        referencedColumnNames: [migrationColumn.referencedTableColumnName],
+        referencedTableName: migrationColumn.referencedTableName,
+        onDelete: 'CASCADE',
+      }),
+    );
+
+    // Create unique constraint if for one to one relation
+    if (migrationColumn.isUnique) {
+      await queryRunner.createUniqueConstraint(
+        `${schemaName}.${tableName}`,
+        new TableUnique({
+          name: `UNIQUE_${tableName}_${migrationColumn.columnName}`,
+          columnNames: [migrationColumn.columnName],
+        }),
+      );
+    }
+  }
+}