Improve sync performances (#12639)

## Goal

We have identified that sync-metadata (which is called during new
workspace initialization) is slow mainly because of workspaceMigration
application (migration-runner module). This is due to the fact that we
use typeORM API to perform schema changes, which often query the
existing schema. As querying the existing schema is costly (especially
with ~1M existing columns) and as we already have what we need described
as metadata, we will use raw SQL directly. This should divide the
workspace initialization time by x2.

## How

This PR can be read in two commits:

1. Extract functions tied to column migrations in a separate service
(`workspace-migration-column.service`) + deprecate COMMENT column
migration type which is not useful since we are not using pg-graphql
anymore
2. Re-work `workspace-migration-column.service` to make it clearer + use
raw SQL

## Result

Before:
<img width="1367" alt="image"
src="https://github.com/user-attachments/assets/e730df7b-db7f-4433-9ce5-52841b010990"
/>

After:
<img width="1367" alt="image"
src="https://github.com/user-attachments/assets/72d2c2b1-2475-4541-a3d5-50b70824a2e4"
/>



## Manual Testing

- Sync-metadata OK
- Workspace init OK
This commit is contained in:
Charles Bochet
2025-06-16 23:53:42 +02:00
committed by GitHub
parent 497faca574
commit d1e0af7f38
9 changed files with 460 additions and 483 deletions

View File

@ -14,7 +14,6 @@ export enum WorkspaceMigrationColumnActionType {
CREATE_FOREIGN_KEY = 'CREATE_FOREIGN_KEY',
DROP_FOREIGN_KEY = 'DROP_FOREIGN_KEY',
DROP = 'DROP',
CREATE_COMMENT = 'CREATE_COMMENT',
}
export type WorkspaceMigrationRenamedEnum = { from: string; to: string };
export type WorkspaceMigrationEnum = string | WorkspaceMigrationRenamedEnum;
@ -57,16 +56,15 @@ export type WorkspaceMigrationColumnAlter = {
alteredColumnDefinition: WorkspaceMigrationColumnDefinition;
};
export type WorkspaceMigrationColumnCreateRelation = {
export type WorkspaceMigrationColumnCreateForeignKey = {
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY;
columnName: string;
referencedTableName: string;
referencedTableColumnName: string;
isUnique?: boolean;
onDelete?: RelationOnDeleteAction;
};
export type WorkspaceMigrationColumnDropRelation = {
export type WorkspaceMigrationColumnDropForeignKey = {
action: WorkspaceMigrationColumnActionType.DROP_FOREIGN_KEY;
columnName: string;
};
@ -76,11 +74,6 @@ export type WorkspaceMigrationColumnDrop = {
columnName: string;
};
export type WorkspaceMigrationCreateComment = {
action: WorkspaceMigrationColumnActionType.CREATE_COMMENT;
comment: string;
};
export type WorkspaceMigrationForeignColumnDefinition =
WorkspaceMigrationColumnDefinition & {
distantColumnName: string;
@ -108,10 +101,9 @@ export type WorkspaceMigrationColumnAction = {
} & (
| WorkspaceMigrationColumnCreate
| WorkspaceMigrationColumnAlter
| WorkspaceMigrationColumnCreateRelation
| WorkspaceMigrationColumnDropRelation
| WorkspaceMigrationColumnCreateForeignKey
| WorkspaceMigrationColumnDropForeignKey
| WorkspaceMigrationColumnDrop
| WorkspaceMigrationCreateComment
);
/**

View File

@ -173,7 +173,6 @@ export class WorkspaceMigrationFieldRelationFactory {
referencedTableName:
computeObjectTargetTable(targetObjectMetadata),
referencedTableColumnName: 'id',
isUnique: false,
onDelete: sourceFieldMetadata.settings.onDelete,
},
],
@ -280,7 +279,6 @@ export class WorkspaceMigrationFieldRelationFactory {
referencedTableName:
computeObjectTargetTable(targetObjectMetadata),
referencedTableColumnName: 'id',
isUnique: false,
onDelete: sourceFieldMetadata.settings.onDelete,
},
],

View File

@ -0,0 +1,370 @@
import { Injectable, Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { QueryRunner, TableColumn } from 'typeorm';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
WorkspaceMigrationColumnAlter,
WorkspaceMigrationColumnCreate,
WorkspaceMigrationColumnCreateForeignKey,
WorkspaceMigrationColumnDrop,
WorkspaceMigrationColumnDropForeignKey,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service';
import { WorkspaceMigrationTypeService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-type.service';
import { PostgresQueryRunner } from 'src/engine/workspace-manager/workspace-migration-runner/types/postgres-query-runner.type';
import { computePostgresEnumName } from 'src/engine/workspace-manager/workspace-migration-runner/utils/compute-postgres-enum-name.util';
import { convertOnDeleteActionToOnDelete } from 'src/engine/workspace-manager/workspace-migration-runner/utils/convert-on-delete-action-to-on-delete.util';
import { typeormBuildCreateColumnSql } from 'src/engine/workspace-manager/workspace-migration-runner/utils/internal/typeorm-build-create-column-sql.util';
import { removeSqlDDLInjection } from 'src/engine/workspace-manager/workspace-migration-runner/utils/remove-sql-injection.util';
@Injectable()
export class WorkspaceMigrationColumnService {
private readonly logger = new Logger(WorkspaceMigrationColumnService.name);
constructor(
private readonly workspaceMigrationEnumService: WorkspaceMigrationEnumService,
private readonly workspaceMigrationTypeService: WorkspaceMigrationTypeService,
) {}
public async handleColumnChanges(
queryRunner: PostgresQueryRunner,
schemaName: string,
tableName: string,
columnMigrations?: WorkspaceMigrationColumnAction[],
) {
if (!columnMigrations || columnMigrations.length === 0) {
return;
}
const columnsByAction = this.groupColumnsByAction(columnMigrations);
if (columnsByAction.create.length > 0) {
await this.handleCreateColumns(
queryRunner,
schemaName,
tableName,
columnsByAction.create,
);
}
if (columnsByAction.drop.length > 0) {
await this.handleDropColumns(
queryRunner,
schemaName,
tableName,
columnsByAction.drop,
);
}
if (columnsByAction.alter.length > 0) {
for (const column of columnsByAction.alter) {
await this.alterColumn(queryRunner, schemaName, tableName, column);
}
}
if (columnsByAction.createForeignKey.length > 0) {
for (const column of columnsByAction.createForeignKey) {
await this.createForeignKey(queryRunner, schemaName, tableName, column);
}
}
if (columnsByAction.dropForeignKey.length > 0) {
for (const column of columnsByAction.dropForeignKey) {
await this.dropForeignKey(queryRunner, schemaName, tableName, column);
}
}
}
private groupColumnsByAction(
columnMigrations: WorkspaceMigrationColumnAction[],
) {
return columnMigrations.reduce(
(acc, column) => {
switch (column.action) {
case WorkspaceMigrationColumnActionType.CREATE:
acc.create.push(column as WorkspaceMigrationColumnCreate);
break;
case WorkspaceMigrationColumnActionType.ALTER:
acc.alter.push(column as WorkspaceMigrationColumnAlter);
break;
case WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY:
acc.createForeignKey.push(
column as WorkspaceMigrationColumnCreateForeignKey,
);
break;
case WorkspaceMigrationColumnActionType.DROP_FOREIGN_KEY:
acc.dropForeignKey.push(
column as WorkspaceMigrationColumnDropForeignKey,
);
break;
case WorkspaceMigrationColumnActionType.DROP:
acc.drop.push(column as WorkspaceMigrationColumnDrop);
break;
}
return acc;
},
{
create: [] as WorkspaceMigrationColumnCreate[],
alter: [] as WorkspaceMigrationColumnAlter[],
createForeignKey: [] as WorkspaceMigrationColumnCreateForeignKey[],
dropForeignKey: [] as WorkspaceMigrationColumnDropForeignKey[],
drop: [] as WorkspaceMigrationColumnDrop[],
},
);
}
public createTableColumnFromMigration(
tableName: string,
column: WorkspaceMigrationColumnCreate,
): TableColumn {
const enumName = column.enum?.length
? `${tableName}_${column.columnName}_enum`
: undefined;
return new TableColumn({
name: column.columnName,
type: column.columnType,
default: column.defaultValue,
isPrimary: column.columnName === 'id',
enum: column.enum?.filter(
(value): value is string => typeof value === 'string',
),
enumName: enumName,
isArray: column.isArray,
isNullable: column.isNullable,
asExpression: column.asExpression,
generatedType: column.generatedType,
});
}
public async handleCreateColumns(
queryRunner: PostgresQueryRunner,
schemaName: string,
tableName: string,
createColumnMigrations: WorkspaceMigrationColumnCreate[],
) {
for (const createColumnMigration of createColumnMigrations) {
// TODO: remove once refactor is complete, id column is already created during table creation
if (createColumnMigration.columnName === 'id') {
continue;
}
if (isDefined(createColumnMigration.enum)) {
const enumName = computePostgresEnumName({
tableName,
columnName: createColumnMigration.columnName,
});
const joinedEnumValues = createColumnMigration.enum
.map((value) => removeSqlDDLInjection(value.toString()))
.map((value) => `'${value}'`)
.join(',');
// TODO: remove once drop table as been refactored to remove the enum types
await queryRunner.query(`DROP TYPE IF EXISTS "${enumName}"`);
await queryRunner.query(
`CREATE TYPE "${enumName}" AS ENUM (${joinedEnumValues})`,
);
}
await queryRunner.query(
`ALTER TABLE "${schemaName}"."${tableName}" ADD ${typeormBuildCreateColumnSql(
{
table: { name: tableName },
column: {
name: createColumnMigration.columnName,
type: createColumnMigration.columnType,
isArray: createColumnMigration.isArray ?? false,
isNullable: createColumnMigration.isNullable,
default: createColumnMigration.defaultValue,
asExpression: createColumnMigration.asExpression,
generatedType: createColumnMigration.generatedType,
},
},
)}`,
);
if (createColumnMigration.columnName === 'id') {
const pkName = queryRunner.connection.namingStrategy.primaryKeyName(
tableName,
[createColumnMigration.columnName],
);
await queryRunner.query(
`ALTER TABLE "${schemaName}"."${tableName} ADD CONSTRAINT "${pkName}" PRIMARY KEY (${removeSqlDDLInjection(createColumnMigration.columnName)})`,
);
}
}
}
public async handleDropColumns(
queryRunner: PostgresQueryRunner,
schemaName: string,
tableName: string,
dropColumns: WorkspaceMigrationColumnDrop[],
) {
if (dropColumns.length === 0) return;
const columnNames = dropColumns.map((column) => column.columnName);
await queryRunner.dropColumns(`${schemaName}.${tableName}`, columnNames);
}
public async alterColumn(
queryRunner: PostgresQueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnAlter,
) {
const enumValues = migrationColumn.alteredColumnDefinition.enum;
// TODO: Maybe we can do something better if we can recreate the old `TableColumn` object
if (enumValues) {
// This is returning the old enum values to avoid TypeORM dropping the enum type
await this.workspaceMigrationEnumService.alterEnum(
queryRunner,
schemaName,
tableName,
migrationColumn,
);
return;
}
if (
migrationColumn.currentColumnDefinition.columnType !==
migrationColumn.alteredColumnDefinition.columnType
) {
await this.workspaceMigrationTypeService.alterType(
queryRunner,
schemaName,
tableName,
migrationColumn,
);
migrationColumn.currentColumnDefinition.columnType =
migrationColumn.alteredColumnDefinition.columnType;
return;
}
await queryRunner.changeColumn(
`${schemaName}.${tableName}`,
new TableColumn({
name: migrationColumn.currentColumnDefinition.columnName,
type: migrationColumn.currentColumnDefinition.columnType,
default: migrationColumn.currentColumnDefinition.defaultValue,
enum: migrationColumn.currentColumnDefinition.enum?.filter(
(value): value is string => typeof value === 'string',
),
isArray: migrationColumn.currentColumnDefinition.isArray,
isNullable: migrationColumn.currentColumnDefinition.isNullable,
/* For now unique constraints are created at a higher level
as we need to handle soft-delete and a bug on empty strings
*/
// isUnique: migrationColumn.currentColumnDefinition.isUnique,
}),
new TableColumn({
name: migrationColumn.alteredColumnDefinition.columnName,
type: migrationColumn.alteredColumnDefinition.columnType,
default: migrationColumn.alteredColumnDefinition.defaultValue,
enum: migrationColumn.currentColumnDefinition.enum?.filter(
(value): value is string => typeof value === 'string',
),
isArray: migrationColumn.alteredColumnDefinition.isArray,
isNullable: migrationColumn.alteredColumnDefinition.isNullable,
asExpression: migrationColumn.alteredColumnDefinition.asExpression,
generatedType: migrationColumn.alteredColumnDefinition.generatedType,
/* For now unique constraints are created at a higher level
as we need to handle soft-delete and a bug on empty strings
*/
// isUnique: migrationColumn.alteredColumnDefinition.isUnique,
}),
);
}
private async createForeignKey(
queryRunner: PostgresQueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnCreateForeignKey,
) {
// Code reference:
// https://github.com/typeorm/typeorm/blob/master/src/driver/postgres/PostgresQueryRunner.ts#L2894
const foreignKeyName = queryRunner.connection.namingStrategy.foreignKeyName(
tableName,
[migrationColumn.columnName],
`${schemaName}.${migrationColumn.referencedTableName}`,
[migrationColumn.referencedTableColumnName],
);
let sql =
`ALTER TABLE "${schemaName}"."${tableName}" ADD CONSTRAINT "${
foreignKeyName
}" FOREIGN KEY ("${removeSqlDDLInjection(migrationColumn.columnName)}") ` +
`REFERENCES "${schemaName}"."${removeSqlDDLInjection(migrationColumn.referencedTableName)}"("${removeSqlDDLInjection(migrationColumn.referencedTableColumnName)}")`;
if (migrationColumn.onDelete)
sql += ` ON DELETE ${convertOnDeleteActionToOnDelete(migrationColumn.onDelete)}`;
await queryRunner.query(sql);
}
private async dropForeignKey(
queryRunner: PostgresQueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnDropForeignKey,
) {
const foreignKeyName = await this.getForeignKeyName(
queryRunner,
schemaName,
tableName,
migrationColumn.columnName,
);
if (!foreignKeyName) {
throw new Error(
`Foreign key not found for column ${migrationColumn.columnName}`,
);
}
await queryRunner.dropForeignKey(
`${schemaName}.${tableName}`,
foreignKeyName,
);
}
private async getForeignKeyName(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
columnName: string,
): Promise<string | undefined> {
const foreignKeys = await queryRunner.query(
`
SELECT
tc.constraint_name AS constraint_name
FROM
information_schema.table_constraints AS tc
JOIN
information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE
tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = $1
AND tc.table_name = $2
AND kcu.column_name = $3
`,
[schemaName, tableName, columnName],
);
return foreignKeys[0]?.constraint_name;
}
}

View File

@ -0,0 +1,9 @@
import { QueryRunner } from 'typeorm';
export type PostgresQueryRunner = QueryRunner & {
connection: QueryRunner['connection'] & {
driver: QueryRunner['connection']['driver'] & {
uuidGenerator: () => string;
};
};
};

View File

@ -0,0 +1,13 @@
import { removeSqlDDLInjection } from 'src/engine/workspace-manager/workspace-migration-runner/utils/remove-sql-injection.util';
type ComputePostgresEnumNameParams = {
tableName: string;
columnName: string;
};
export const computePostgresEnumName = ({
tableName,
columnName,
}: ComputePostgresEnumNameParams): string => {
return removeSqlDDLInjection(`${tableName}_${columnName}_enum`);
};

View File

@ -0,0 +1,45 @@
// This code is from TypeORM and is used to build the SQL for creating a column
import { Table, TableColumn } from 'typeorm';
import { computePostgresEnumName } from 'src/engine/workspace-manager/workspace-migration-runner/utils/compute-postgres-enum-name.util';
// Do not modify: https://github.com/typeorm/typeorm/blob/master/src/driver/postgres/PostgresQueryRunner.ts#L4713
export const typeormBuildCreateColumnSql = ({
table,
column,
}: {
table: Pick<Table, 'name'>;
column: Pick<
TableColumn,
| 'name'
| 'type'
| 'isArray'
| 'isNullable'
| 'default'
| 'generatedType'
| 'asExpression'
>;
}): string => {
let columnSql = '"' + column.name + '"';
if (column.type === 'enum' || column.type === 'simple-enum') {
columnSql += ` "${computePostgresEnumName({
tableName: table.name,
columnName: column.name,
})}"`;
if (column.isArray) columnSql += ' array';
} else {
columnSql += ' ' + column.type;
}
if (column.generatedType === 'STORED' && column.asExpression) {
columnSql += ` GENERATED ALWAYS AS (${column.asExpression}) STORED`;
}
if (column.isNullable !== true) columnSql += ' NOT NULL';
if (column.default !== undefined && column.default !== null)
columnSql += ' DEFAULT ' + column.default;
return columnSql;
};

View File

@ -0,0 +1,3 @@
export const removeSqlDDLInjection = (value: string): string => {
return value.replace(/[^a-zA-Z0-9_]/g, '');
};

View File

@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceMigrationColumnService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-column.service';
import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service';
import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.service';
@ -14,6 +15,7 @@ import { WorkspaceMigrationTypeService } from './services/workspace-migration-ty
WorkspaceMigrationRunnerService,
WorkspaceMigrationEnumService,
WorkspaceMigrationTypeService,
WorkspaceMigrationColumnService,
],
exports: [WorkspaceMigrationRunnerService],
})

View File

@ -1,24 +1,13 @@
import { Injectable, Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import {
QueryRunner,
Table,
TableColumn,
TableForeignKey,
TableIndex,
TableUnique,
} from 'typeorm';
import { QueryRunner, Table, TableColumn, TableIndex } from 'typeorm';
import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
WorkspaceMigrationColumnAlter,
WorkspaceMigrationColumnCreate,
WorkspaceMigrationColumnCreateRelation,
WorkspaceMigrationColumnDrop,
WorkspaceMigrationColumnDropRelation,
WorkspaceMigrationForeignTable,
WorkspaceMigrationIndexAction,
WorkspaceMigrationIndexActionType,
@ -27,12 +16,10 @@ import {
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service';
import { convertOnDeleteActionToOnDelete } from 'src/engine/workspace-manager/workspace-migration-runner/utils/convert-on-delete-action-to-on-delete.util';
import { WorkspaceMigrationColumnService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-column.service';
import { PostgresQueryRunner } from 'src/engine/workspace-manager/workspace-migration-runner/types/postgres-query-runner.type';
import { tableDefaultColumns } from 'src/engine/workspace-manager/workspace-migration-runner/utils/table-default-column.util';
import { WorkspaceMigrationTypeService } from './services/workspace-migration-type.service';
export const RELATION_MIGRATION_PRIORITY_PREFIX = '1000';
@Injectable()
@ -42,16 +29,9 @@ export class WorkspaceMigrationRunnerService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationEnumService: WorkspaceMigrationEnumService,
private readonly workspaceMigrationTypeService: WorkspaceMigrationTypeService,
private readonly workspaceMigrationColumnService: WorkspaceMigrationColumnService,
) {}
/**
* Executes pending migrations for a given workspace
*
* @param workspaceId string
* @returns Promise<WorkspaceMigrationTableAction[]>
*/
public async executeMigrationFromPendingMigrations(
workspaceId: string,
): Promise<WorkspaceMigrationTableAction[]> {
@ -74,7 +54,8 @@ export class WorkspaceMigrationRunnerService {
return [...acc, ...pendingMigration.migrations];
}, []);
const queryRunner = mainDataSource.createQueryRunner();
const queryRunner =
mainDataSource.createQueryRunner() as PostgresQueryRunner;
await queryRunner.connect();
await queryRunner.startTransaction();
@ -115,15 +96,8 @@ export class WorkspaceMigrationRunnerService {
return flattenedPendingMigrations;
}
/**
* Handles table changes for a given migration
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableMigration WorkspaceMigrationTableAction
*/
private async handleTableChanges(
queryRunner: QueryRunner,
queryRunner: PostgresQueryRunner,
schemaName: string,
tableMigration: WorkspaceMigrationTableAction,
) {
@ -149,7 +123,7 @@ export class WorkspaceMigrationRunnerService {
}
if (tableMigration.columns && tableMigration.columns.length > 0) {
await this.handleColumnChanges(
await this.workspaceMigrationColumnService.handleColumnChanges(
queryRunner,
schemaName,
tableMigration.newName ?? tableMigration.name,
@ -203,14 +177,6 @@ export class WorkspaceMigrationRunnerService {
}
}
/**
* Handles index changes for a given table
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
* @param indexes WorkspaceMigrationIndexAction[]
*/
private async handleIndexesChanges(
queryRunner: QueryRunner,
schemaName: string,
@ -231,14 +197,6 @@ export class WorkspaceMigrationRunnerService {
}
}
/**
* Creates an index on a table
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
* @param index WorkspaceMigrationIndexAction
*/
private async createIndex(
queryRunner: QueryRunner,
schemaName: string,
@ -272,14 +230,6 @@ export class WorkspaceMigrationRunnerService {
}
}
/**
* Drops an index from a table
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
* @param indexName string
*/
private async dropIndex(
queryRunner: QueryRunner,
schemaName: string,
@ -300,16 +250,8 @@ export class WorkspaceMigrationRunnerService {
}
}
/**
* Creates a table with columns from migration
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
* @param columns WorkspaceMigrationColumnAction[]
*/
private async createTable(
queryRunner: QueryRunner,
queryRunner: PostgresQueryRunner,
schemaName: string,
tableName: string,
columns?: WorkspaceMigrationColumnAction[],
@ -323,7 +265,10 @@ export class WorkspaceMigrationRunnerService {
for (const column of createColumns) {
tableColumns.push(
this.createTableColumnFromMigration(tableName, column),
this.workspaceMigrationColumnService.createTableColumnFromMigration(
tableName,
column,
),
);
}
}
@ -343,7 +288,7 @@ export class WorkspaceMigrationRunnerService {
);
if (nonCreateColumns.length > 0) {
await this.handleColumnChanges(
await this.workspaceMigrationColumnService.handleColumnChanges(
queryRunner,
schemaName,
tableName,
@ -353,44 +298,6 @@ export class WorkspaceMigrationRunnerService {
}
}
/**
* Creates a TableColumn object from a migration column
*
* @param tableName string
* @param column WorkspaceMigrationColumnCreate
* @returns TableColumn
*/
private createTableColumnFromMigration(
tableName: string,
column: WorkspaceMigrationColumnCreate,
): TableColumn {
const enumName = column.enum?.length
? `${tableName}_${column.columnName}_enum`
: undefined;
return new TableColumn({
name: column.columnName,
type: column.columnType,
default: column.defaultValue,
isPrimary: column.columnName === 'id',
enum: column.enum?.filter(
(value): value is string => typeof value === 'string',
),
enumName: enumName,
isArray: column.isArray,
isNullable: column.isNullable,
asExpression: column.asExpression,
generatedType: column.generatedType,
});
}
/**
* Rename a table
* @param queryRunner QueryRunner
* @param schemaName string
* @param oldTableName string
* @param newTableName string
*/
private async renameTable(
queryRunner: QueryRunner,
schemaName: string,
@ -403,368 +310,6 @@ export class WorkspaceMigrationRunnerService {
);
}
/**
* Handles column changes for a given migration
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
* @param columnMigrations WorkspaceMigrationColumnAction[]
*/
private async handleColumnChanges(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
columnMigrations?: WorkspaceMigrationColumnAction[],
) {
if (!columnMigrations || columnMigrations.length === 0) {
return;
}
const columnsByAction = this.groupColumnsByAction(columnMigrations);
if (columnsByAction.create.length > 0) {
await this.handleCreateColumns(
queryRunner,
schemaName,
tableName,
columnsByAction.create,
);
}
if (columnsByAction.drop.length > 0) {
await this.handleDropColumns(
queryRunner,
schemaName,
tableName,
columnsByAction.drop,
);
}
await this.handleOtherColumnActions(
queryRunner,
schemaName,
tableName,
columnsByAction.alter,
columnsByAction.createForeignKey,
columnsByAction.dropForeignKey,
columnsByAction.createComment,
);
}
private groupColumnsByAction(
columnMigrations: WorkspaceMigrationColumnAction[],
) {
return columnMigrations.reduce(
(acc, column) => {
switch (column.action) {
case WorkspaceMigrationColumnActionType.CREATE:
acc.create.push(column as WorkspaceMigrationColumnCreate);
break;
case WorkspaceMigrationColumnActionType.ALTER:
acc.alter.push(column as WorkspaceMigrationColumnAlter);
break;
case WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY:
acc.createForeignKey.push(
column as WorkspaceMigrationColumnCreateRelation,
);
break;
case WorkspaceMigrationColumnActionType.DROP_FOREIGN_KEY:
acc.dropForeignKey.push(
column as WorkspaceMigrationColumnDropRelation,
);
break;
case WorkspaceMigrationColumnActionType.DROP:
acc.drop.push(column as WorkspaceMigrationColumnDrop);
break;
case WorkspaceMigrationColumnActionType.CREATE_COMMENT:
acc.createComment.push(
column as {
action: WorkspaceMigrationColumnActionType.CREATE_COMMENT;
comment: string;
},
);
break;
}
return acc;
},
{
create: [] as WorkspaceMigrationColumnCreate[],
alter: [] as WorkspaceMigrationColumnAlter[],
createForeignKey: [] as WorkspaceMigrationColumnCreateRelation[],
dropForeignKey: [] as WorkspaceMigrationColumnDropRelation[],
drop: [] as WorkspaceMigrationColumnDrop[],
createComment: [] as {
action: WorkspaceMigrationColumnActionType.CREATE_COMMENT;
comment: string;
}[],
},
);
}
private async handleCreateColumns(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
createColumns: WorkspaceMigrationColumnCreate[],
) {
if (createColumns.length === 0) return;
const table = await queryRunner.getTable(`${schemaName}.${tableName}`);
if (!table) {
throw new Error(`Table "${tableName}" not found`);
}
const existingColumns = new Set(table.columns.map((column) => column.name));
const columnsToCreate = createColumns.filter(
(column) => !existingColumns.has(column.columnName),
);
if (columnsToCreate.length === 0) return;
const tableColumns = columnsToCreate.map((column) =>
this.createTableColumnFromMigration(tableName, column),
);
await queryRunner.addColumns(`${schemaName}.${tableName}`, tableColumns);
}
private async handleDropColumns(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
dropColumns: WorkspaceMigrationColumnDrop[],
) {
if (dropColumns.length === 0) return;
const columnNames = dropColumns.map((column) => column.columnName);
await queryRunner.dropColumns(`${schemaName}.${tableName}`, columnNames);
}
private async handleOtherColumnActions(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
alterColumns: WorkspaceMigrationColumnAlter[],
createForeignKeyColumns: WorkspaceMigrationColumnCreateRelation[],
dropForeignKeyColumns: WorkspaceMigrationColumnDropRelation[],
createCommentColumns: {
action: WorkspaceMigrationColumnActionType.CREATE_COMMENT;
comment: string;
}[],
) {
for (const column of alterColumns) {
await this.alterColumn(queryRunner, schemaName, tableName, column);
}
for (const column of createForeignKeyColumns) {
await this.createRelation(queryRunner, schemaName, tableName, column);
}
for (const column of dropForeignKeyColumns) {
await this.dropRelation(queryRunner, schemaName, tableName, column);
}
for (const column of createCommentColumns) {
await this.createComment(
queryRunner,
schemaName,
tableName,
column.comment,
);
}
}
private async alterColumn(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnAlter,
) {
const enumValues = migrationColumn.alteredColumnDefinition.enum;
// TODO: Maybe we can do something better if we can recreate the old `TableColumn` object
if (enumValues) {
// This is returning the old enum values to avoid TypeORM dropping the enum type
await this.workspaceMigrationEnumService.alterEnum(
queryRunner,
schemaName,
tableName,
migrationColumn,
);
return;
}
if (
migrationColumn.currentColumnDefinition.columnType !==
migrationColumn.alteredColumnDefinition.columnType
) {
await this.workspaceMigrationTypeService.alterType(
queryRunner,
schemaName,
tableName,
migrationColumn,
);
migrationColumn.currentColumnDefinition.columnType =
migrationColumn.alteredColumnDefinition.columnType;
return;
}
await queryRunner.changeColumn(
`${schemaName}.${tableName}`,
new TableColumn({
name: migrationColumn.currentColumnDefinition.columnName,
type: migrationColumn.currentColumnDefinition.columnType,
default: migrationColumn.currentColumnDefinition.defaultValue,
enum: migrationColumn.currentColumnDefinition.enum?.filter(
(value): value is string => typeof value === 'string',
),
isArray: migrationColumn.currentColumnDefinition.isArray,
isNullable: migrationColumn.currentColumnDefinition.isNullable,
/* For now unique constraints are created at a higher level
as we need to handle soft-delete and a bug on empty strings
*/
// isUnique: migrationColumn.currentColumnDefinition.isUnique,
}),
new TableColumn({
name: migrationColumn.alteredColumnDefinition.columnName,
type: migrationColumn.alteredColumnDefinition.columnType,
default: migrationColumn.alteredColumnDefinition.defaultValue,
enum: migrationColumn.currentColumnDefinition.enum?.filter(
(value): value is string => typeof value === 'string',
),
isArray: migrationColumn.alteredColumnDefinition.isArray,
isNullable: migrationColumn.alteredColumnDefinition.isNullable,
asExpression: migrationColumn.alteredColumnDefinition.asExpression,
generatedType: migrationColumn.alteredColumnDefinition.generatedType,
/* For now unique constraints are created at a higher level
as we need to handle soft-delete and a bug on empty strings
*/
// isUnique: migrationColumn.alteredColumnDefinition.isUnique,
}),
);
}
private async createRelation(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnCreateRelation,
) {
try {
await queryRunner.createForeignKey(
`${schemaName}.${tableName}`,
new TableForeignKey({
columnNames: [migrationColumn.columnName],
referencedColumnNames: [migrationColumn.referencedTableColumnName],
referencedTableName: migrationColumn.referencedTableName,
referencedSchema: schemaName,
onDelete: convertOnDeleteActionToOnDelete(migrationColumn.onDelete),
}),
);
// TODO remove me after 0.53 release @prastoin @charlesBochet Swallowing blocking false positive constraint
} catch (error) {
if (
[error.driverError.message, error.message]
.filter(isDefined)
.some((el: string) => el.includes('FK_e078063f0cbce9767a0f8ca431d'))
) {
this.logger.warn(
'Encountered a FK_e078063f0cbce9767a0f8ca431d exception, swallowing',
);
} else {
throw error;
}
}
/// End remove me
// Create unique constraint if for one to one relation
if (migrationColumn.isUnique) {
await queryRunner.createUniqueConstraint(
`${schemaName}.${tableName}`,
new TableUnique({
name: `UNIQUE_${tableName}_${migrationColumn.columnName}`,
columnNames: [migrationColumn.columnName],
}),
);
}
}
private async dropRelation(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnDropRelation,
) {
const foreignKeyName = await this.getForeignKeyName(
queryRunner,
schemaName,
tableName,
migrationColumn.columnName,
);
if (!foreignKeyName) {
// Todo: Remove this temporary hack tied to 0.32 upgrade
if (migrationColumn.columnName === 'activityId') {
return;
}
throw new Error(
`Foreign key not found for column ${migrationColumn.columnName}`,
);
}
await queryRunner.dropForeignKey(
`${schemaName}.${tableName}`,
foreignKeyName,
);
}
private async getForeignKeyName(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
columnName: string,
): Promise<string | undefined> {
const foreignKeys = await queryRunner.query(
`
SELECT
tc.constraint_name AS constraint_name
FROM
information_schema.table_constraints AS tc
JOIN
information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE
tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = $1
AND tc.table_name = $2
AND kcu.column_name = $3
`,
[schemaName, tableName, columnName],
);
return foreignKeys[0]?.constraint_name;
}
private async createComment(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
comment: string,
) {
await queryRunner.query(`
COMMENT ON TABLE "${schemaName}"."${tableName}" IS e'${comment}';
`);
}
private async createForeignTable(
queryRunner: QueryRunner,
schemaName: string,