From 7a4599321b10a13710ca1ca28f141af7cc5d57bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Malfait?= Date: Wed, 7 May 2025 17:13:47 +0200 Subject: [PATCH] Prepare for schema fusion (#11922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Architecture Detail The goal is to merge the two TypeORM schemas. Having two schemas prevent doing things like fieldMetadata.workspace in TypeORM, and create useless debates since there is no clear line (is a serverlessFunction core or metadata? What about events? etc.) ### Before ``` ┌───────────────────┐ ┌───────────────────┐ │ core schema │ │ metadata schema │ ├───────────────────┤ ├───────────────────┤ │- User │ │- ObjectMetadata │ │- Workspace │ │- FieldMetadata │ │- UserWorkspace │ │- RelationMetadata │ │- etc. │ │- etc. │ └───────────────────┘ └───────────────────┘ ``` ### After the Migration ``` ┌───────────────────────────────────────────┐ │ engine schema │ ├───────────────────────────────────────────┤ │- User - ObjectMetadata │ │- Workspace - FieldMetadata │ │- UserWorkspace - RelationMetadata │ │- etc. - etc. │ └───────────────────────────────────────────┘ ``` ## Strategy 1. During 0.53 we backfill the *_typeorm_migrations* table of the core schema with all metadata migrations 2. That way in 0.54 we can move the metadata migrations from the metadata folder to the core folder. We will also edit the migration files to reference "core" instead of "metadata". For people doing a fresh install this will run smoothly and create the tables in Core directly. For people on an existing install, this migrations will not run because they were added to the *_typeorm_migrations* in 0.53 3. In 0.55 we will rename "core" to something else (for example "engine") Note: if someone jumps version, for example skips to 0.54 directly without having run 0.53 then this could cause issue. In 0.54 we should consider gating the "migrate:prod" in the docker file so that it's controlled and run by the upgrade command (and not run if the command wasn't executed properly) --- .../0-53-copy-typeorm-migrations.command.ts | 91 +++++++++++++++++++ .../0-53-upgrade-version-command.module.ts | 3 + .../upgrade.command.ts | 3 + 3 files changed, 97 insertions(+) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-copy-typeorm-migrations.command.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-copy-typeorm-migrations.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-copy-typeorm-migrations.command.ts new file mode 100644 index 000000000..03be096dc --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-copy-typeorm-migrations.command.ts @@ -0,0 +1,91 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; + +import { + ActiveOrSuspendedWorkspacesMigrationCommandRunner, + RunOnWorkspaceArgs, +} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; + +@Command({ + name: 'upgrade:0-53:copy-typeorm-migrations', + description: 'Copy _typeorm_migrations from metadata schema to core schema', +}) +export class CopyTypeormMigrationsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + async runOnWorkspace(args: RunOnWorkspaceArgs): Promise { + // This command doesn't need to run per workspace, only once + if (args.index !== 0) { + return; + } + + this.logger.log( + 'Starting to copy _typeorm_migrations from metadata to core', + ); + + const queryRunner = + this.workspaceRepository.manager.connection.createQueryRunner(); + + try { + await queryRunner.connect(); + await queryRunner.startTransaction(); + + const metadataMigrations = await queryRunner.query( + 'SELECT * FROM metadata._typeorm_migrations ORDER BY id ASC', + ); + + this.logger.log( + `Found ${metadataMigrations.length} migrations in metadata schema`, + ); + + if (args.options?.dryRun) { + this.logger.log('Dry run mode - no changes will be applied'); + + return; + } + + const existingCoreMigrations = await queryRunner.query( + 'SELECT name FROM core._typeorm_migrations', + ); + + const existingMigrationNames = new Set( + existingCoreMigrations.map((migration) => migration.name), + ); + + for (const migration of metadataMigrations) { + if (!existingMigrationNames.has(migration.name)) { + await queryRunner.query( + 'INSERT INTO core._typeorm_migrations ("timestamp", name) VALUES ($1, $2)', + [migration.timestamp, migration.name], + ); + this.logger.log(`Copied migration: ${migration.name}`); + } else { + this.logger.log( + `Migration ${migration.name} already exists in core schema`, + ); + } + } + + await queryRunner.commitTransaction(); + this.logger.log( + 'Successfully copied all migrations from metadata to core schema', + ); + } catch (error) { + await queryRunner.rollbackTransaction(); + this.logger.error(`Failed to copy migrations: ${error.message}`); + throw error; + } finally { + await queryRunner.release(); + } + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-upgrade-version-command.module.ts index f153e98ee..c2bcac09f 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/0-53/0-53-upgrade-version-command.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { BackfillWorkflowNextStepIdsCommand } from 'src/database/commands/upgrade-version-command/0-53/0-53-backfill-workflow-next-step-ids.command'; +import { CopyTypeormMigrationsCommand } from 'src/database/commands/upgrade-version-command/0-53/0-53-copy-typeorm-migrations.command'; import { MigrateWorkflowEventListenersToAutomatedTriggersCommand } from 'src/database/commands/upgrade-version-command/0-53/0-53-migrate-workflow-event-listeners-to-automated-triggers.command'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; @@ -14,10 +15,12 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works providers: [ MigrateWorkflowEventListenersToAutomatedTriggersCommand, BackfillWorkflowNextStepIdsCommand, + CopyTypeormMigrationsCommand, ], exports: [ MigrateWorkflowEventListenersToAutomatedTriggersCommand, BackfillWorkflowNextStepIdsCommand, + CopyTypeormMigrationsCommand, ], }) export class V0_53_UpgradeVersionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index 45746e5a0..0a08d5fa6 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -19,6 +19,7 @@ import { UpgradeCreatedByEnumCommand } from 'src/database/commands/upgrade-versi import { MigrateRelationsToFieldMetadataCommand } from 'src/database/commands/upgrade-version-command/0-52/0-52-migrate-relations-to-field-metadata.command'; import { UpgradeDateAndDateTimeFieldsSettingsJsonCommand } from 'src/database/commands/upgrade-version-command/0-52/0-52-upgrade-settings-field'; import { BackfillWorkflowNextStepIdsCommand } from 'src/database/commands/upgrade-version-command/0-53/0-53-backfill-workflow-next-step-ids.command'; +import { CopyTypeormMigrationsCommand } from 'src/database/commands/upgrade-version-command/0-53/0-53-copy-typeorm-migrations.command'; import { MigrateWorkflowEventListenersToAutomatedTriggersCommand } from 'src/database/commands/upgrade-version-command/0-53/0-53-migrate-workflow-event-listeners-to-automated-triggers.command'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; @@ -60,6 +61,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { // 0.53 Commands protected readonly migrateWorkflowEventListenersToAutomatedTriggersCommand: MigrateWorkflowEventListenersToAutomatedTriggersCommand, protected readonly backfillWorkflowNextStepIdsCommand: BackfillWorkflowNextStepIdsCommand, + protected readonly copyTypeormMigrationsCommand: CopyTypeormMigrationsCommand, ) { super( workspaceRepository, @@ -111,6 +113,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { afterSyncMetadata: [ this.migrateWorkflowEventListenersToAutomatedTriggersCommand, this.backfillWorkflowNextStepIdsCommand, + this.copyTypeormMigrationsCommand, ], };