From 0dc1cd9df13c2e99a86befc2e8cddab2a83f13f7 Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:44:41 +0100 Subject: [PATCH] [REG/FEAT] Sync metadata after upgrade (#10557) # Introduction Historically we've been programmatically running sync metadata just after all upgrade command's migration. Adding back this behavior as default to the new dynamic modules Duplicated already existing synchronize metadata logic as a quick fix as we're about to iterate over commands next sprint --- .../create-upgrade-all-command.factory.ts | 80 ++++++++++++++++++- .../migration-command.module.ts | 10 ++- .../services/sync-workspace-logger.module.ts | 9 +++ .../services/sync-workspace-logger.service.ts | 4 +- ...workspace-sync-metadata-commands.module.ts | 10 +-- 5 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts diff --git a/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts b/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts index 4d95da353..5eb173585 100644 --- a/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts +++ b/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts @@ -1,11 +1,19 @@ import { Inject } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; import { MigrationCommandInterface } from 'src/database/commands/migration-command/interfaces/migration-command.interface'; +import { MaintainedWorkspacesMigrationCommandRunner } from 'src/database/commands/migration-command/maintained-workspaces-migration-command.runner'; import { MIGRATION_COMMAND_INJECTION_TOKEN } from 'src/database/commands/migration-command/migration-command.constants'; import { MigrationCommandRunner } from 'src/database/commands/migration-command/migration-command.runner'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { SyncWorkspaceLoggerService } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service'; +import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service'; export function createUpgradeAllCommand( version: string, @@ -14,17 +22,81 @@ export function createUpgradeAllCommand( name: `upgrade-${version}`, description: `Upgrade to version ${version}`, }) - class UpgradeCommand extends MigrationCommandRunner { + class UpgradeCommand extends MaintainedWorkspacesMigrationCommandRunner { constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, @Inject(MIGRATION_COMMAND_INJECTION_TOKEN) private readonly subCommands: MigrationCommandInterface[], + private readonly dataSourceService: DataSourceService, + private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService, + private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService, ) { - super(); + super(workspaceRepository, twentyORMGlobalManager); } - async runMigrationCommand( + // TODO Remove and avoid duplicated synchronize logic with SyncWorkspaceMetadataCommand after command refactoring + private async synchronizeWorkspaceMetadata({ + workspaceIds, + options, + }: { + workspaceIds: string[]; + options: Record; + }) { + this.logger.log(`Attempting to sync ${workspaceIds.length} workspaces.`); + const errorsDuringSync: string[] = []; + + for (const [index, workspaceId] of workspaceIds.entries()) { + try { + this.logger.log( + `Running workspace sync for workspace: ${workspaceId} (${index + 1} out of ${workspaceIds.length})`, + ); + const dataSourceMetadata = + await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( + workspaceId, + ); + + const { storage, workspaceMigrations } = + await this.workspaceSyncMetadataService.synchronize( + { + workspaceId, + dataSourceId: dataSourceMetadata.id, + }, + { applyChanges: !options.dryRun }, + ); + + if (options.dryRun) { + await this.syncWorkspaceLoggerService.saveLogs( + workspaceId, + storage, + workspaceMigrations, + ); + } + } catch (error) { + const errorMessage = `Failed to synchronize workspace ${workspaceId}: ${error.message}`; + + this.logger.error(errorMessage); + errorsDuringSync.push(errorMessage); + + continue; + } + } + this.logger.log( + `Finished synchronizing all active workspaces (${ + workspaceIds.length + } workspaces). ${ + errorsDuringSync.length > 0 + ? 'Errors during sync:\n' + errorsDuringSync.join('.\n') + : '' + }`, + ); + } + + async runMigrationCommandOnMaintainedWorkspaces( passedParams: string[], options: Record, + workspaceIds: string[], ): Promise { this.logger.log(`Running upgrade command for version ${version}`); @@ -32,6 +104,8 @@ export function createUpgradeAllCommand( await command.runMigrationCommand(passedParams, options); } + await this.synchronizeWorkspaceMetadata({ options, workspaceIds }); + this.logger.log(`Upgrade ${version} command completed!`); } } diff --git a/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts b/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts index 404245d5f..d4958249a 100644 --- a/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts +++ b/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts @@ -5,6 +5,9 @@ import { MigrationCommandInterface } from 'src/database/commands/migration-comma import { createUpgradeAllCommand } from 'src/database/commands/migration-command/create-upgrade-all-command.factory'; import { getMigrationCommandsForVersion } from 'src/database/commands/migration-command/decorators/migration-command.decorator'; import { MIGRATION_COMMAND_INJECTION_TOKEN } from 'src/database/commands/migration-command/migration-command.constants'; +import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; +import { SyncWorkspaceLoggerModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module'; +import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module'; @Module({}) export class MigrationCommandModule { @@ -17,7 +20,12 @@ export class MigrationCommandModule { return { module: MigrationCommandModule, - imports: moduleMetadata.imports, + imports: [ + SyncWorkspaceLoggerModule, + ...(moduleMetadata.imports ?? []), + WorkspaceSyncMetadataModule, + DataSourceModule, + ], providers: [ ...(moduleMetadata.providers ?? []), ...commandClasses, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts new file mode 100644 index 000000000..2ac0ec229 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; + +import { SyncWorkspaceLoggerService } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service'; + +@Module({ + providers: [SyncWorkspaceLoggerService], + exports: [SyncWorkspaceLoggerService], +}) +export class SyncWorkspaceLoggerModule {} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts index 25f92347d..f587a3294 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts @@ -1,8 +1,8 @@ import { Injectable } from '@nestjs/common'; -import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; -import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { CommandLogger } from 'src/command/command-logger'; +import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; @Injectable() export class SyncWorkspaceLoggerService { diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts index 6ae43c404..5a753043c 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts @@ -8,11 +8,10 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module'; import { ConvertRecordPositionsToIntegers } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/convert-record-positions-to-integers.command'; import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module'; +import { SyncWorkspaceLoggerModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module'; import { SyncWorkspaceMetadataCommand } from './sync-workspace-metadata.command'; -import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service'; - @Module({ imports: [ WorkspaceSyncMetadataModule, @@ -21,12 +20,9 @@ import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.ser DataSourceModule, WorkspaceDataSourceModule, TypeOrmModule.forFeature([Workspace], 'core'), + SyncWorkspaceLoggerModule, ], - providers: [ - SyncWorkspaceMetadataCommand, - ConvertRecordPositionsToIntegers, - SyncWorkspaceLoggerService, - ], + providers: [SyncWorkspaceMetadataCommand, ConvertRecordPositionsToIntegers], exports: [SyncWorkspaceMetadataCommand], }) export class WorkspaceSyncMetadataCommandsModule {}