Improve migration runner performances (#10572)
## Context Workspace creation and more specifically sync-metadata performances are bad at the moment. We are trying to identify bottlenecks and one of the root causes is the migration runner that can take up to 10s when setting up a new workspaces with all its tables. First observation is we do a lot of things sequentially, mostly to make the code easier to read and debug but it impacts performances. For example, a table creation is done in two steps, we first ask typeorm to create the table then ask typeorm to create columns (and sometimes columns one by one), each instruction can take time because typeorm seems to do some checks internally. The proposition here is to try to merge migrations when possible, for example when we create a table we want the migration to also contain the columns it will contain so we can ask typeorm to add the columns at the same time. We are also using batch operations when possible (addColumns instead of addColumn, dropColumns instead of dropColumn) Still, we could go further with foreign keys creations or/and try with raw query directly. ## Test New workspace creation: See screenshot, 9865.40233296156ms is on main, the rest is after the changes: <img width="610" alt="Screenshot 2025-02-28 at 09 27 21" src="https://github.com/user-attachments/assets/42e880ff-279e-4170-b705-009e4b72045c" /> ResetDB and Sync-metadata on an existing workspace commands still work
This commit is contained in:
Weiko
@ -5,6 +5,7 @@ import { DataSource } from 'typeorm';
|
||||
import { NodeEnvironment } from 'src/engine/core-modules/environment/interfaces/node-environment.interface';
|
||||
|
||||
import { AppToken } from 'src/engine/core-modules/app-token/app-token.entity';
|
||||
import { ApprovedAccessDomain } from 'src/engine/core-modules/approved-access-domain/approved-access-domain.entity';
|
||||
import { BillingCustomer } from 'src/engine/core-modules/billing/entities/billing-customer.entity';
|
||||
import { BillingEntitlement } from 'src/engine/core-modules/billing/entities/billing-entitlement.entity';
|
||||
import { BillingMeter } from 'src/engine/core-modules/billing/entities/billing-meter.entity';
|
||||
@ -22,7 +23,6 @@ import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-works
|
||||
import { User } from 'src/engine/core-modules/user/user.entity';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||
import { ApprovedAccessDomain } from 'src/engine/core-modules/approved-access-domain/approved-access-domain.entity';
|
||||
@Injectable()
|
||||
export class TypeORMService implements OnModuleInit, OnModuleDestroy {
|
||||
private mainDataSource: DataSource;
|
||||
@ -33,7 +33,10 @@ export class TypeORMService implements OnModuleInit, OnModuleDestroy {
|
||||
this.mainDataSource = new DataSource({
|
||||
url: environmentService.get('PG_DATABASE_URL'),
|
||||
type: 'postgres',
|
||||
logging: false,
|
||||
logging:
|
||||
environmentService.get('NODE_ENV') === NodeEnvironment.development
|
||||
? ['query', 'error']
|
||||
: ['error'],
|
||||
schema: 'core',
|
||||
entities: [
|
||||
User,
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { isDefined } from 'class-validator';
|
||||
import {
|
||||
@ -17,6 +17,7 @@ import {
|
||||
WorkspaceMigrationColumnAlter,
|
||||
WorkspaceMigrationColumnCreate,
|
||||
WorkspaceMigrationColumnCreateRelation,
|
||||
WorkspaceMigrationColumnDrop,
|
||||
WorkspaceMigrationColumnDropRelation,
|
||||
WorkspaceMigrationForeignTable,
|
||||
WorkspaceMigrationIndexAction,
|
||||
@ -34,6 +35,8 @@ import { WorkspaceMigrationTypeService } from './services/workspace-migration-ty
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceMigrationRunnerService {
|
||||
private readonly logger = new Logger(WorkspaceMigrationRunnerService.name);
|
||||
|
||||
constructor(
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
private readonly workspaceMigrationService: WorkspaceMigrationService,
|
||||
@ -89,8 +92,11 @@ export class WorkspaceMigrationRunnerService {
|
||||
|
||||
await queryRunner.commitTransaction();
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error('Error executing migration', error);
|
||||
this.logger.error(
|
||||
`Error executing migration: ${error.message}`,
|
||||
error.stack,
|
||||
);
|
||||
|
||||
await queryRunner.rollbackTransaction();
|
||||
throw error;
|
||||
} finally {
|
||||
@ -114,7 +120,7 @@ export class WorkspaceMigrationRunnerService {
|
||||
*
|
||||
* @param queryRunner QueryRunner
|
||||
* @param schemaName string
|
||||
* @param tableMigration WorkspaceMigrationTableChange
|
||||
* @param tableMigration WorkspaceMigrationTableAction
|
||||
*/
|
||||
private async handleTableChanges(
|
||||
queryRunner: QueryRunner,
|
||||
@ -138,6 +144,8 @@ export class WorkspaceMigrationRunnerService {
|
||||
tableMigration.name,
|
||||
tableMigration.newName,
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if (tableMigration.columns && tableMigration.columns.length > 0) {
|
||||
@ -147,6 +155,8 @@ export class WorkspaceMigrationRunnerService {
|
||||
tableMigration.newName ?? tableMigration.name,
|
||||
tableMigration.columns,
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
@ -193,6 +203,14 @@ export class WorkspaceMigrationRunnerService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles index changes for a given table
|
||||
*
|
||||
* @param queryRunner QueryRunner
|
||||
* @param schemaName string
|
||||
* @param tableName string
|
||||
* @param indexes WorkspaceMigrationIndexAction[]
|
||||
*/
|
||||
private async handleIndexesChanges(
|
||||
queryRunner: QueryRunner,
|
||||
schemaName: string,
|
||||
@ -202,48 +220,10 @@ export class WorkspaceMigrationRunnerService {
|
||||
for (const index of indexes) {
|
||||
switch (index.action) {
|
||||
case WorkspaceMigrationIndexActionType.CREATE:
|
||||
try {
|
||||
if (isDefined(index.type) && index.type !== IndexType.BTREE) {
|
||||
const quotedColumns = index.columns.map(
|
||||
(column) => `"${column}"`,
|
||||
);
|
||||
|
||||
await queryRunner.query(`
|
||||
CREATE INDEX "${index.name}" ON "${schemaName}"."${tableName}" USING ${index.type} (${quotedColumns.join(', ')})
|
||||
`);
|
||||
} else {
|
||||
await queryRunner.createIndex(
|
||||
`${schemaName}.${tableName}`,
|
||||
new TableIndex({
|
||||
name: index.name,
|
||||
columnNames: index.columns,
|
||||
isUnique: index.isUnique,
|
||||
where: index.where ?? undefined,
|
||||
}),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore error if index already exists
|
||||
if (error.code === '42P07') {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
await this.createIndex(queryRunner, schemaName, tableName, index);
|
||||
break;
|
||||
case WorkspaceMigrationIndexActionType.DROP:
|
||||
try {
|
||||
await queryRunner.dropIndex(
|
||||
`${schemaName}.${tableName}`,
|
||||
index.name,
|
||||
);
|
||||
} catch (error) {
|
||||
// Ignore error if index does not exist
|
||||
if (
|
||||
error.message ===
|
||||
`Supplied index ${index.name} was not found in table ${schemaName}.${tableName}`
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
await this.dropIndex(queryRunner, schemaName, tableName, index.name);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Migration index action not supported`);
|
||||
@ -252,11 +232,81 @@ export class WorkspaceMigrationRunnerService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a table for a given schema and table name
|
||||
* Creates an index on a table
|
||||
*
|
||||
* @param queryRunner QueryRunner
|
||||
* @param schemaName string
|
||||
* @param tableName string
|
||||
* @param index WorkspaceMigrationIndexAction
|
||||
*/
|
||||
private async createIndex(
|
||||
queryRunner: QueryRunner,
|
||||
schemaName: string,
|
||||
tableName: string,
|
||||
index: WorkspaceMigrationIndexAction,
|
||||
) {
|
||||
try {
|
||||
if (isDefined(index.type) && index.type !== IndexType.BTREE) {
|
||||
const quotedColumns = index.columns.map((column) => `"${column}"`);
|
||||
|
||||
await queryRunner.query(`
|
||||
CREATE INDEX "${index.name}" ON "${schemaName}"."${tableName}" USING ${index.type} (${quotedColumns.join(', ')})
|
||||
`);
|
||||
} else {
|
||||
await queryRunner.createIndex(
|
||||
`${schemaName}.${tableName}`,
|
||||
new TableIndex({
|
||||
name: index.name,
|
||||
columnNames: index.columns,
|
||||
isUnique: index.isUnique,
|
||||
where: index.where ?? undefined,
|
||||
}),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore error if index already exists
|
||||
if (error.code === '42P07') {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops an index from a table
|
||||
*
|
||||
* @param queryRunner QueryRunner
|
||||
* @param schemaName string
|
||||
* @param tableName string
|
||||
* @param indexName string
|
||||
*/
|
||||
private async dropIndex(
|
||||
queryRunner: QueryRunner,
|
||||
schemaName: string,
|
||||
tableName: string,
|
||||
indexName: string,
|
||||
) {
|
||||
try {
|
||||
await queryRunner.dropIndex(`${schemaName}.${tableName}`, indexName);
|
||||
} catch (error) {
|
||||
// Ignore error if index does not exist
|
||||
if (
|
||||
error.message ===
|
||||
`Supplied index ${indexName} was not found in table ${schemaName}.${tableName}`
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a table with columns from migration
|
||||
*
|
||||
* @param queryRunner QueryRunner
|
||||
* @param schemaName string
|
||||
* @param tableName string
|
||||
* @param columns WorkspaceMigrationColumnAction[]
|
||||
*/
|
||||
private async createTable(
|
||||
queryRunner: QueryRunner,
|
||||
@ -264,25 +314,76 @@ export class WorkspaceMigrationRunnerService {
|
||||
tableName: string,
|
||||
columns?: WorkspaceMigrationColumnAction[],
|
||||
) {
|
||||
const tableColumns: TableColumn[] = [];
|
||||
|
||||
if (columns && columns.length > 0) {
|
||||
const createColumns = columns.filter(
|
||||
(column) => column.action === WorkspaceMigrationColumnActionType.CREATE,
|
||||
) as WorkspaceMigrationColumnCreate[];
|
||||
|
||||
for (const column of createColumns) {
|
||||
tableColumns.push(
|
||||
this.createTableColumnFromMigration(tableName, column),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await queryRunner.createTable(
|
||||
new Table({
|
||||
name: tableName,
|
||||
schema: schemaName,
|
||||
columns: tableDefaultColumns(),
|
||||
columns: tableColumns.length > 0 ? tableColumns : tableDefaultColumns(),
|
||||
}),
|
||||
true,
|
||||
);
|
||||
|
||||
if (columns && columns.length > 0) {
|
||||
await this.handleColumnChanges(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columns,
|
||||
const nonCreateColumns = columns.filter(
|
||||
(column) => column.action !== WorkspaceMigrationColumnActionType.CREATE,
|
||||
);
|
||||
|
||||
if (nonCreateColumns.length > 0) {
|
||||
await this.handleColumnChanges(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
nonCreateColumns,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a TableColumn object from a migration column
|
||||
*
|
||||
* @param tableName string
|
||||
* @param column WorkspaceMigrationColumnCreate
|
||||
* @returns TableColumn
|
||||
*/
|
||||
private createTableColumnFromMigration(
|
||||
tableName: string,
|
||||
column: WorkspaceMigrationColumnCreate,
|
||||
): TableColumn {
|
||||
const enumName = column.enum?.length
|
||||
? `${tableName}_${column.columnName}_enum`
|
||||
: undefined;
|
||||
|
||||
return new TableColumn({
|
||||
name: column.columnName,
|
||||
type: column.columnType,
|
||||
default: column.defaultValue,
|
||||
isPrimary: column.columnName === 'id',
|
||||
enum: column.enum?.filter(
|
||||
(value): value is string => typeof value === 'string',
|
||||
),
|
||||
enumName: enumName,
|
||||
isArray: column.isArray,
|
||||
isNullable: column.isNullable,
|
||||
asExpression: column.asExpression,
|
||||
generatedType: column.generatedType,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename a table
|
||||
* @param queryRunner QueryRunner
|
||||
@ -309,7 +410,6 @@ export class WorkspaceMigrationRunnerService {
|
||||
* @param schemaName string
|
||||
* @param tableName string
|
||||
* @param columnMigrations WorkspaceMigrationColumnAction[]
|
||||
* @returns
|
||||
*/
|
||||
private async handleColumnChanges(
|
||||
queryRunner: QueryRunner,
|
||||
@ -321,105 +421,162 @@ export class WorkspaceMigrationRunnerService {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const columnMigration of columnMigrations) {
|
||||
switch (columnMigration.action) {
|
||||
case WorkspaceMigrationColumnActionType.CREATE:
|
||||
await this.createColumn(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnMigration,
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.ALTER:
|
||||
await this.alterColumn(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnMigration,
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY:
|
||||
await this.createRelation(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnMigration,
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.DROP_FOREIGN_KEY:
|
||||
await this.dropRelation(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnMigration,
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.DROP:
|
||||
await queryRunner.dropColumn(
|
||||
`${schemaName}.${tableName}`,
|
||||
columnMigration.columnName,
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.CREATE_COMMENT:
|
||||
await this.createComment(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnMigration.comment,
|
||||
);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Migration column action not supported`);
|
||||
}
|
||||
const columnsByAction = this.groupColumnsByAction(columnMigrations);
|
||||
|
||||
if (columnsByAction.create.length > 0) {
|
||||
await this.handleCreateColumns(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnsByAction.create,
|
||||
);
|
||||
}
|
||||
|
||||
if (columnsByAction.drop.length > 0) {
|
||||
await this.handleDropColumns(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnsByAction.drop,
|
||||
);
|
||||
}
|
||||
|
||||
await this.handleOtherColumnActions(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
columnsByAction.alter,
|
||||
columnsByAction.createForeignKey,
|
||||
columnsByAction.dropForeignKey,
|
||||
columnsByAction.createComment,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a column for a given schema, table name, and column migration
|
||||
*
|
||||
* @param queryRunner QueryRunner
|
||||
* @param schemaName string
|
||||
* @param tableName string
|
||||
* @param migrationColumn WorkspaceMigrationColumnAction
|
||||
*/
|
||||
private async createColumn(
|
||||
private groupColumnsByAction(
|
||||
columnMigrations: WorkspaceMigrationColumnAction[],
|
||||
) {
|
||||
return columnMigrations.reduce(
|
||||
(acc, column) => {
|
||||
switch (column.action) {
|
||||
case WorkspaceMigrationColumnActionType.CREATE:
|
||||
acc.create.push(column as WorkspaceMigrationColumnCreate);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.ALTER:
|
||||
acc.alter.push(column as WorkspaceMigrationColumnAlter);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY:
|
||||
acc.createForeignKey.push(
|
||||
column as WorkspaceMigrationColumnCreateRelation,
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.DROP_FOREIGN_KEY:
|
||||
acc.dropForeignKey.push(
|
||||
column as WorkspaceMigrationColumnDropRelation,
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.DROP:
|
||||
acc.drop.push(column as WorkspaceMigrationColumnDrop);
|
||||
break;
|
||||
case WorkspaceMigrationColumnActionType.CREATE_COMMENT:
|
||||
acc.createComment.push(
|
||||
column as {
|
||||
action: WorkspaceMigrationColumnActionType.CREATE_COMMENT;
|
||||
comment: string;
|
||||
},
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
create: [] as WorkspaceMigrationColumnCreate[],
|
||||
alter: [] as WorkspaceMigrationColumnAlter[],
|
||||
createForeignKey: [] as WorkspaceMigrationColumnCreateRelation[],
|
||||
dropForeignKey: [] as WorkspaceMigrationColumnDropRelation[],
|
||||
drop: [] as WorkspaceMigrationColumnDrop[],
|
||||
createComment: [] as {
|
||||
action: WorkspaceMigrationColumnActionType.CREATE_COMMENT;
|
||||
comment: string;
|
||||
}[],
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async handleCreateColumns(
|
||||
queryRunner: QueryRunner,
|
||||
schemaName: string,
|
||||
tableName: string,
|
||||
migrationColumn: WorkspaceMigrationColumnCreate,
|
||||
createColumns: WorkspaceMigrationColumnCreate[],
|
||||
) {
|
||||
const hasColumn = await queryRunner.hasColumn(
|
||||
`${schemaName}.${tableName}`,
|
||||
migrationColumn.columnName,
|
||||
);
|
||||
if (createColumns.length === 0) return;
|
||||
|
||||
if (hasColumn) {
|
||||
return;
|
||||
const table = await queryRunner.getTable(`${schemaName}.${tableName}`);
|
||||
|
||||
if (!table) {
|
||||
throw new Error(`Table "${tableName}" not found`);
|
||||
}
|
||||
|
||||
const enumName = `${tableName}_${migrationColumn.columnName}_enum`;
|
||||
const existingColumns = new Set(table.columns.map((column) => column.name));
|
||||
|
||||
await queryRunner.addColumn(
|
||||
`${schemaName}.${tableName}`,
|
||||
new TableColumn({
|
||||
name: migrationColumn.columnName,
|
||||
type: migrationColumn.columnType,
|
||||
default: migrationColumn.defaultValue,
|
||||
enum: migrationColumn.enum?.filter(
|
||||
(value): value is string => typeof value === 'string',
|
||||
),
|
||||
enumName: enumName,
|
||||
isArray: migrationColumn.isArray,
|
||||
isNullable: migrationColumn.isNullable,
|
||||
/* For now unique constraints are created at a higher level
|
||||
as we need to handle soft-delete and a bug on empty strings
|
||||
*/
|
||||
// isUnique: migrationColumn.isUnique,
|
||||
asExpression: migrationColumn.asExpression,
|
||||
generatedType: migrationColumn.generatedType,
|
||||
}),
|
||||
const columnsToCreate = createColumns.filter(
|
||||
(column) => !existingColumns.has(column.columnName),
|
||||
);
|
||||
|
||||
if (columnsToCreate.length === 0) return;
|
||||
|
||||
const tableColumns = columnsToCreate.map((column) =>
|
||||
this.createTableColumnFromMigration(tableName, column),
|
||||
);
|
||||
|
||||
await queryRunner.addColumns(`${schemaName}.${tableName}`, tableColumns);
|
||||
}
|
||||
|
||||
private async handleDropColumns(
|
||||
queryRunner: QueryRunner,
|
||||
schemaName: string,
|
||||
tableName: string,
|
||||
dropColumns: WorkspaceMigrationColumnDrop[],
|
||||
) {
|
||||
if (dropColumns.length === 0) return;
|
||||
|
||||
const columnNames = dropColumns.map((column) => column.columnName);
|
||||
|
||||
await queryRunner.dropColumns(`${schemaName}.${tableName}`, columnNames);
|
||||
}
|
||||
|
||||
private async handleOtherColumnActions(
|
||||
queryRunner: QueryRunner,
|
||||
schemaName: string,
|
||||
tableName: string,
|
||||
alterColumns: WorkspaceMigrationColumnAlter[],
|
||||
createForeignKeyColumns: WorkspaceMigrationColumnCreateRelation[],
|
||||
dropForeignKeyColumns: WorkspaceMigrationColumnDropRelation[],
|
||||
createCommentColumns: {
|
||||
action: WorkspaceMigrationColumnActionType.CREATE_COMMENT;
|
||||
comment: string;
|
||||
}[],
|
||||
) {
|
||||
for (const column of alterColumns) {
|
||||
await this.alterColumn(queryRunner, schemaName, tableName, column);
|
||||
}
|
||||
|
||||
for (const column of createForeignKeyColumns) {
|
||||
await this.createRelation(queryRunner, schemaName, tableName, column);
|
||||
}
|
||||
|
||||
for (const column of dropForeignKeyColumns) {
|
||||
await this.dropRelation(queryRunner, schemaName, tableName, column);
|
||||
}
|
||||
|
||||
for (const column of createCommentColumns) {
|
||||
await this.createComment(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableName,
|
||||
column.comment,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async alterColumn(
|
||||
|
||||
@ -2,22 +2,22 @@ import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
|
||||
import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
|
||||
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
|
||||
import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
|
||||
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
|
||||
import { mapObjectMetadataByUniqueIdentifier } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/sync-metadata.util';
|
||||
import { StandardRelationFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-relation.factory';
|
||||
import { WorkspaceRelationComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-relation.comparator';
|
||||
import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service';
|
||||
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
import { WorkspaceMigrationRelationFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-relation.factory';
|
||||
import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects';
|
||||
import { CustomWorkspaceEntity } from 'src/engine/twenty-orm/custom.workspace-entity';
|
||||
import { WorkspaceMigrationRelationFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-relation.factory';
|
||||
import { WorkspaceRelationComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-relation.comparator';
|
||||
import { StandardRelationFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-relation.factory';
|
||||
import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service';
|
||||
import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects';
|
||||
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
import { mapObjectMetadataByUniqueIdentifier } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/sync-metadata.util';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceSyncRelationMetadataService {
|
||||
|
||||
@ -1,14 +1,16 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
|
||||
import { InjectDataSource } from '@nestjs/typeorm';
|
||||
|
||||
import { DataSource, QueryFailedError, Repository } from 'typeorm';
|
||||
import { DataSource, QueryFailedError } from 'typeorm';
|
||||
|
||||
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
|
||||
import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import {
|
||||
WorkspaceMigrationEntity,
|
||||
WorkspaceMigrationTableActionType,
|
||||
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
|
||||
import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service';
|
||||
import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service';
|
||||
@ -36,8 +38,6 @@ export class WorkspaceSyncMetadataService {
|
||||
private readonly workspaceSyncIndexMetadataService: WorkspaceSyncIndexMetadataService,
|
||||
private readonly workspaceSyncObjectMetadataIdentifiersService: WorkspaceSyncObjectMetadataIdentifiersService,
|
||||
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
|
||||
@InjectRepository(FeatureFlag, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlag>,
|
||||
) {}
|
||||
|
||||
/**
|
||||
@ -112,6 +112,15 @@ export class WorkspaceSyncMetadataService {
|
||||
`Workspace field migrations took ${workspaceFieldMigrationsEnd - workspaceFieldMigrationsStart}ms`,
|
||||
);
|
||||
|
||||
// Merge object and field migrations during table creation
|
||||
const {
|
||||
objectMigrations: mergedObjectMigrations,
|
||||
fieldMigrations: mergedFieldMigrations,
|
||||
} = this.mergeMigrations({
|
||||
objectMigrations: workspaceObjectMigrations,
|
||||
fieldMigrations: workspaceFieldMigrations,
|
||||
});
|
||||
|
||||
// 3 - Sync standard relations on standard and custom objects
|
||||
const workspaceRelationMigrationsStart = performance.now();
|
||||
const workspaceRelationMigrations =
|
||||
@ -164,8 +173,8 @@ export class WorkspaceSyncMetadataService {
|
||||
|
||||
// Save workspace migrations into the database
|
||||
workspaceMigrations = await workspaceMigrationRepository.save([
|
||||
...workspaceObjectMigrations,
|
||||
...workspaceFieldMigrations,
|
||||
...mergedObjectMigrations,
|
||||
...mergedFieldMigrations,
|
||||
...workspaceRelationMigrations,
|
||||
...workspaceIndexMigrations,
|
||||
]);
|
||||
@ -223,4 +232,62 @@ export class WorkspaceSyncMetadataService {
|
||||
storage,
|
||||
};
|
||||
}
|
||||
|
||||
private mergeMigrations({
|
||||
objectMigrations,
|
||||
fieldMigrations,
|
||||
}: {
|
||||
objectMigrations: Partial<WorkspaceMigrationEntity>[];
|
||||
fieldMigrations: Partial<WorkspaceMigrationEntity>[];
|
||||
}): {
|
||||
objectMigrations: Partial<WorkspaceMigrationEntity>[];
|
||||
fieldMigrations: Partial<WorkspaceMigrationEntity>[];
|
||||
} {
|
||||
const createMigrationsByTable = new Map<string, any>();
|
||||
|
||||
for (const objectMigration of objectMigrations) {
|
||||
if (
|
||||
!objectMigration.migrations ||
|
||||
objectMigration.migrations.length === 0
|
||||
)
|
||||
continue;
|
||||
|
||||
const tableMigration = objectMigration.migrations[0];
|
||||
|
||||
if (tableMigration.action === WorkspaceMigrationTableActionType.CREATE) {
|
||||
createMigrationsByTable.set(tableMigration.name, tableMigration);
|
||||
}
|
||||
}
|
||||
|
||||
const fieldMigrationsWithoutTableCreation = fieldMigrations.filter(
|
||||
(fieldMigration) => {
|
||||
if (
|
||||
!fieldMigration.migrations ||
|
||||
fieldMigration.migrations.length === 0
|
||||
)
|
||||
return true;
|
||||
|
||||
const tableMigration = fieldMigration.migrations[0];
|
||||
const tableName = tableMigration.name;
|
||||
|
||||
if (createMigrationsByTable.has(tableName)) {
|
||||
const createMigration = createMigrationsByTable.get(tableName);
|
||||
|
||||
if (tableMigration.columns?.length) {
|
||||
createMigration.columns = createMigration.columns || [];
|
||||
createMigration.columns.push(...tableMigration.columns);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
objectMigrations,
|
||||
fieldMigrations: fieldMigrationsWithoutTableCreation,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user