feat: refactor folder structure (#4498)

* feat: wip refactor folder structure

* Fix

* fix position

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Jérémy M
2024-03-15 14:40:58 +01:00
committed by GitHub
parent 52f1b3ac98
commit 94487f6737
760 changed files with 3215 additions and 3155 deletions

View File

@ -0,0 +1,37 @@
import { Command, CommandRunner, Option } from 'nest-commander';
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
interface ExecuteWorkspaceMigrationsOptions {
workspaceId: string;
}
@Command({
name: 'workspace:apply-pending-migrations',
description: 'Apply pending migrations',
})
export class WorkspaceExecutePendingMigrationsCommand extends CommandRunner {
constructor(
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
) {
super();
}
async run(
_passedParam: string[],
options: ExecuteWorkspaceMigrationsOptions,
): Promise<void> {
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
options.workspaceId,
);
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
import { WorkspaceExecutePendingMigrationsCommand } from './workspace-execute-pending-migrations.command';
@Module({
imports: [WorkspaceMigrationRunnerModule],
providers: [WorkspaceExecutePendingMigrationsCommand],
})
export class WorkspaceMigrationRunnerCommandsModule {}

View File

@ -0,0 +1,192 @@
import { Injectable } from '@nestjs/common';
import { QueryRunner } from 'typeorm';
import { WorkspaceMigrationColumnAlter } from 'src/engine-metadata/workspace-migration/workspace-migration.entity';
import { serializeDefaultValue } from 'src/engine-metadata/field-metadata/utils/serialize-default-value';
@Injectable()
export class WorkspaceMigrationEnumService {
async alterEnum(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnAlter,
) {
const columnDefinition = migrationColumn.alteredColumnDefinition;
const oldEnumTypeName = `${tableName}_${columnDefinition.columnName}_enum`;
const newEnumTypeName = `${tableName}_${columnDefinition.columnName}_enum_new`;
const enumValues =
columnDefinition.enum?.map((enumValue) => {
if (typeof enumValue === 'string') {
return enumValue;
}
return enumValue.to;
}) ?? [];
if (!columnDefinition.isNullable && !columnDefinition.defaultValue) {
columnDefinition.defaultValue = serializeDefaultValue(enumValues[0]);
}
// Create new enum type with new values
await this.createNewEnumType(
newEnumTypeName,
queryRunner,
schemaName,
enumValues,
);
// Temporarily change column type to text
await queryRunner.query(`
ALTER TABLE "${schemaName}"."${tableName}"
ALTER COLUMN "${columnDefinition.columnName}" TYPE TEXT
`);
// Migrate existing values to new values
await this.migrateEnumValues(
queryRunner,
schemaName,
tableName,
migrationColumn,
);
// Update existing rows to handle missing values
await this.handleMissingEnumValues(
queryRunner,
schemaName,
tableName,
migrationColumn,
enumValues,
);
// Alter column type to new enum
await this.updateColumnToNewEnum(
queryRunner,
schemaName,
tableName,
columnDefinition.columnName,
newEnumTypeName,
columnDefinition.defaultValue,
);
// Drop old enum type
await this.dropOldEnumType(queryRunner, schemaName, oldEnumTypeName);
// Rename new enum type to old enum type name
await this.renameEnumType(
queryRunner,
schemaName,
oldEnumTypeName,
newEnumTypeName,
);
}
private async createNewEnumType(
name: string,
queryRunner: QueryRunner,
schemaName: string,
newValues: string[],
) {
const enumValues = newValues
.map((value) => `'${value.replace(/'/g, "''")}'`)
.join(', ');
await queryRunner.query(
`CREATE TYPE "${schemaName}"."${name}" AS ENUM (${enumValues})`,
);
}
private async migrateEnumValues(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnAlter,
) {
const columnDefinition = migrationColumn.alteredColumnDefinition;
if (!columnDefinition.enum) {
return;
}
for (const enumValue of columnDefinition.enum) {
// Skip string values
if (typeof enumValue === 'string') {
continue;
}
await queryRunner.query(`
UPDATE "${schemaName}"."${tableName}"
SET "${columnDefinition.columnName}" = '${enumValue.to}'
WHERE "${columnDefinition.columnName}" = '${enumValue.from}'
`);
}
}
private async handleMissingEnumValues(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnAlter,
enumValues: string[],
) {
const columnDefinition = migrationColumn.alteredColumnDefinition;
// Set missing values to null or default value
let defaultValue = 'NULL';
if (columnDefinition.defaultValue) {
if (Array.isArray(columnDefinition.defaultValue)) {
defaultValue = `ARRAY[${columnDefinition.defaultValue
.map((e) => `'${e}'`)
.join(', ')}]`;
} else {
defaultValue = columnDefinition.defaultValue;
}
}
await queryRunner.query(`
UPDATE "${schemaName}"."${tableName}"
SET "${columnDefinition.columnName}" = ${defaultValue}
WHERE "${columnDefinition.columnName}" NOT IN (${enumValues
.map((e) => `'${e}'`)
.join(', ')})
`);
}
private async updateColumnToNewEnum(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
columnName: string,
newEnumTypeName: string,
newDefaultValue: string,
) {
await queryRunner.query(
`ALTER TABLE "${schemaName}"."${tableName}" ALTER COLUMN "${columnName}" DROP DEFAULT,
ALTER COLUMN "${columnName}" TYPE "${schemaName}"."${newEnumTypeName}" USING ("${columnName}"::text::"${schemaName}"."${newEnumTypeName}"),
ALTER COLUMN "${columnName}" SET DEFAULT ${newDefaultValue}`,
);
}
private async dropOldEnumType(
queryRunner: QueryRunner,
schemaName: string,
oldEnumTypeName: string,
) {
await queryRunner.query(
`DROP TYPE IF EXISTS "${schemaName}"."${oldEnumTypeName}"`,
);
}
private async renameEnumType(
queryRunner: QueryRunner,
schemaName: string,
oldEnumTypeName: string,
newEnumTypeName: string,
) {
await queryRunner.query(`
ALTER TYPE "${schemaName}"."${newEnumTypeName}"
RENAME TO "${oldEnumTypeName}"
`);
}
}

View File

@ -0,0 +1,35 @@
import { Injectable } from '@nestjs/common';
import { QueryRunner } from 'typeorm';
import { WorkspaceMigrationColumnAlter } from 'src/engine-metadata/workspace-migration/workspace-migration.entity';
@Injectable()
export class WorkspaceMigrationTypeService {
constructor() {}
async alterType(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnAlter,
) {
const columnDefinition = migrationColumn.alteredColumnDefinition;
// Update the column type
// If casting is not possible, the query will fail
await queryRunner.query(`
ALTER TABLE "${schemaName}"."${tableName}"
ALTER COLUMN "${columnDefinition.columnName}" TYPE ${columnDefinition.columnType}
USING "${columnDefinition.columnName}"::${columnDefinition.columnType}
`);
// Update the column default value
if (columnDefinition.defaultValue) {
await queryRunner.query(`
ALTER TABLE "${schemaName}"."${tableName}"
ALTER COLUMN "${columnDefinition.columnName}" SET DEFAULT ${columnDefinition.defaultValue}::${columnDefinition.columnType};
`);
}
}
}

View File

@ -0,0 +1,22 @@
import { RelationOnDeleteAction } from 'src/engine-metadata/relation-metadata/relation-metadata.entity';
export const convertOnDeleteActionToOnDelete = (
onDeleteAction: RelationOnDeleteAction | undefined,
): 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION' | undefined => {
if (!onDeleteAction) {
return 'SET NULL';
}
switch (onDeleteAction) {
case 'CASCADE':
return 'CASCADE';
case 'SET_NULL':
return 'SET NULL';
case 'RESTRICT':
return 'RESTRICT';
case 'NO_ACTION':
return 'NO ACTION';
default:
throw new Error('Invalid onDeleteAction');
}
};

View File

@ -0,0 +1,25 @@
import { TableColumnOptions } from 'typeorm';
export const customTableDefaultColumns: TableColumnOptions[] = [
{
name: 'id',
type: 'uuid',
isPrimary: true,
default: 'public.uuid_generate_v4()',
},
{
name: 'createdAt',
type: 'timestamp',
default: 'now()',
},
{
name: 'updatedAt',
type: 'timestamp',
default: 'now()',
},
{
name: 'deletedAt',
type: 'timestamp',
isNullable: true,
},
];

View File

@ -0,0 +1,25 @@
import { Module } from '@nestjs/common';
import { WorkspaceMigrationModule } from 'src/engine-metadata/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceCacheVersionModule } from 'src/engine-metadata/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service';
import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.service';
import { WorkspaceMigrationTypeService } from './services/workspace-migration-type.service';
@Module({
imports: [
WorkspaceDataSourceModule,
WorkspaceMigrationModule,
WorkspaceCacheVersionModule,
],
providers: [
WorkspaceMigrationRunnerService,
WorkspaceMigrationEnumService,
WorkspaceMigrationTypeService,
],
exports: [WorkspaceMigrationRunnerService],
})
export class WorkspaceMigrationRunnerModule {}

View File

@ -0,0 +1,415 @@
import { Injectable } from '@nestjs/common';
import {
QueryRunner,
Table,
TableColumn,
TableForeignKey,
TableUnique,
} from 'typeorm';
import { WorkspaceMigrationService } from 'src/engine-metadata/workspace-migration/workspace-migration.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import {
WorkspaceMigrationTableAction,
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
WorkspaceMigrationColumnCreate,
WorkspaceMigrationColumnCreateRelation,
WorkspaceMigrationColumnAlter,
WorkspaceMigrationColumnDropRelation,
} from 'src/engine-metadata/workspace-migration/workspace-migration.entity';
import { WorkspaceCacheVersionService } from 'src/engine-metadata/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service';
import { convertOnDeleteActionToOnDelete } from 'src/engine/workspace-manager/workspace-migration-runner/utils/convert-on-delete-action-to-on-delete.util';
import { customTableDefaultColumns } from './utils/custom-table-default-column.util';
import { WorkspaceMigrationTypeService } from './services/workspace-migration-type.service';
@Injectable()
export class WorkspaceMigrationRunnerService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMigrationEnumService: WorkspaceMigrationEnumService,
private readonly workspaceMigrationTypeService: WorkspaceMigrationTypeService,
) {}
/**
* Executes pending migrations for a given workspace
*
* @param workspaceId string
* @returns Promise<WorkspaceMigrationTableAction[]>
*/
public async executeMigrationFromPendingMigrations(
workspaceId: string,
): Promise<WorkspaceMigrationTableAction[]> {
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
if (!workspaceDataSource) {
throw new Error('Workspace data source not found');
}
const pendingMigrations =
await this.workspaceMigrationService.getPendingMigrations(workspaceId);
if (pendingMigrations.length === 0) {
return [];
}
const flattenedPendingMigrations: WorkspaceMigrationTableAction[] =
pendingMigrations.reduce((acc, pendingMigration) => {
return [...acc, ...pendingMigration.migrations];
}, []);
const queryRunner = workspaceDataSource?.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
const schemaName =
this.workspaceDataSourceService.getSchemaName(workspaceId);
try {
// Loop over each migration and create or update the table
for (const migration of flattenedPendingMigrations) {
await this.handleTableChanges(queryRunner, schemaName, migration);
}
await queryRunner.commitTransaction();
} catch (error) {
console.error('Error executing migration', error);
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
}
// Update appliedAt date for each migration
// TODO: Should be done after the migration is successful
for (const pendingMigration of pendingMigrations) {
await this.workspaceMigrationService.setAppliedAtForMigration(
workspaceId,
pendingMigration,
);
}
// Increment workspace cache version
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
return flattenedPendingMigrations;
}
/**
* Handles table changes for a given migration
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableMigration WorkspaceMigrationTableChange
*/
private async handleTableChanges(
queryRunner: QueryRunner,
schemaName: string,
tableMigration: WorkspaceMigrationTableAction,
) {
switch (tableMigration.action) {
case 'create':
await this.createTable(queryRunner, schemaName, tableMigration.name);
break;
case 'alter':
await this.handleColumnChanges(
queryRunner,
schemaName,
tableMigration.name,
tableMigration?.columns,
);
break;
case 'drop':
await queryRunner.dropTable(`${schemaName}.${tableMigration.name}`);
break;
default:
throw new Error(
`Migration table action ${tableMigration.action} not supported`,
);
}
}
/**
* Creates a table for a given schema and table name
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
*/
private async createTable(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
) {
await queryRunner.createTable(
new Table({
name: tableName,
schema: schemaName,
columns: customTableDefaultColumns,
}),
true,
);
// Enable totalCount for the table
await queryRunner.query(`
COMMENT ON TABLE "${schemaName}"."${tableName}" IS '@graphql({"totalCount": {"enabled": true}})';
`);
}
/**
* Handles column changes for a given migration
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
* @param columnMigrations WorkspaceMigrationColumnAction[]
* @returns
*/
private async handleColumnChanges(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
columnMigrations?: WorkspaceMigrationColumnAction[],
) {
if (!columnMigrations || columnMigrations.length === 0) {
return;
}
for (const columnMigration of columnMigrations) {
switch (columnMigration.action) {
case WorkspaceMigrationColumnActionType.CREATE:
await this.createColumn(
queryRunner,
schemaName,
tableName,
columnMigration,
);
break;
case WorkspaceMigrationColumnActionType.ALTER:
await this.alterColumn(
queryRunner,
schemaName,
tableName,
columnMigration,
);
break;
case WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY:
await this.createRelation(
queryRunner,
schemaName,
tableName,
columnMigration,
);
break;
case WorkspaceMigrationColumnActionType.DROP_FOREIGN_KEY:
await this.dropRelation(
queryRunner,
schemaName,
tableName,
columnMigration,
);
break;
case WorkspaceMigrationColumnActionType.DROP:
await queryRunner.dropColumn(
`${schemaName}.${tableName}`,
columnMigration.columnName,
);
break;
default:
throw new Error(`Migration column action not supported`);
}
}
}
/**
* Creates a column for a given schema, table name, and column migration
*
* @param queryRunner QueryRunner
* @param schemaName string
* @param tableName string
* @param migrationColumn WorkspaceMigrationColumnAction
*/
private async createColumn(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnCreate,
) {
const hasColumn = await queryRunner.hasColumn(
`${schemaName}.${tableName}`,
migrationColumn.columnName,
);
if (hasColumn) {
return;
}
await queryRunner.addColumn(
`${schemaName}.${tableName}`,
new TableColumn({
name: migrationColumn.columnName,
type: migrationColumn.columnType,
default: migrationColumn.defaultValue,
enum: migrationColumn.enum?.filter(
(value): value is string => typeof value === 'string',
),
isArray: migrationColumn.isArray,
isNullable: migrationColumn.isNullable,
}),
);
}
private async alterColumn(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnAlter,
) {
const enumValues = migrationColumn.alteredColumnDefinition.enum;
// TODO: Maybe we can do something better if we can recreate the old `TableColumn` object
if (enumValues) {
// This is returning the old enum values to avoid TypeORM dropping the enum type
await this.workspaceMigrationEnumService.alterEnum(
queryRunner,
schemaName,
tableName,
migrationColumn,
);
return;
}
if (
migrationColumn.currentColumnDefinition.columnType !==
migrationColumn.alteredColumnDefinition.columnType
) {
await this.workspaceMigrationTypeService.alterType(
queryRunner,
schemaName,
tableName,
migrationColumn,
);
migrationColumn.currentColumnDefinition.columnType =
migrationColumn.alteredColumnDefinition.columnType;
return;
}
await queryRunner.changeColumn(
`${schemaName}.${tableName}`,
new TableColumn({
name: migrationColumn.currentColumnDefinition.columnName,
type: migrationColumn.currentColumnDefinition.columnType,
default: migrationColumn.currentColumnDefinition.defaultValue,
enum: migrationColumn.currentColumnDefinition.enum?.filter(
(value): value is string => typeof value === 'string',
),
isArray: migrationColumn.currentColumnDefinition.isArray,
isNullable: migrationColumn.currentColumnDefinition.isNullable,
}),
new TableColumn({
name: migrationColumn.alteredColumnDefinition.columnName,
type: migrationColumn.alteredColumnDefinition.columnType,
default: migrationColumn.alteredColumnDefinition.defaultValue,
enum: migrationColumn.currentColumnDefinition.enum?.filter(
(value): value is string => typeof value === 'string',
),
isArray: migrationColumn.alteredColumnDefinition.isArray,
isNullable: migrationColumn.alteredColumnDefinition.isNullable,
}),
);
}
private async createRelation(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnCreateRelation,
) {
await queryRunner.createForeignKey(
`${schemaName}.${tableName}`,
new TableForeignKey({
columnNames: [migrationColumn.columnName],
referencedColumnNames: [migrationColumn.referencedTableColumnName],
referencedTableName: migrationColumn.referencedTableName,
referencedSchema: schemaName,
onDelete: convertOnDeleteActionToOnDelete(migrationColumn.onDelete),
}),
);
// Create unique constraint if for one to one relation
if (migrationColumn.isUnique) {
await queryRunner.createUniqueConstraint(
`${schemaName}.${tableName}`,
new TableUnique({
name: `UNIQUE_${tableName}_${migrationColumn.columnName}`,
columnNames: [migrationColumn.columnName],
}),
);
}
}
private async dropRelation(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: WorkspaceMigrationColumnDropRelation,
) {
const foreignKeyName = await this.getForeignKeyName(
queryRunner,
schemaName,
tableName,
migrationColumn.columnName,
);
if (!foreignKeyName) {
throw new Error(
`Foreign key not found for column ${migrationColumn.columnName}`,
);
}
await queryRunner.dropForeignKey(
`${schemaName}.${tableName}`,
foreignKeyName,
);
}
private async getForeignKeyName(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
columnName: string,
): Promise<string | undefined> {
const foreignKeys = await queryRunner.query(
`
SELECT
tc.constraint_name AS constraint_name
FROM
information_schema.table_constraints AS tc
JOIN
information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE
tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = $1
AND tc.table_name = $2
AND kcu.column_name = $3
`,
[schemaName, tableName, columnName],
);
return foreignKeys[0]?.constraint_name;
}
}