admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add option to synchronize all active workspaces at once (#6221)

Browse Source 
In the longer term, we want to improve the efficiency and reliability of
the sync-metadata command, by choosing an error handling strategy and
paying greater attention to health checks.
In the meantime, this PR adds an option to run the sync-metadata command
on all active workspaces at once.

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Marie
2024-07-11 19:48:07 +02:00
committed by GitHub
parent 5cb7f68b7c
commit 5ad287baf5
11 changed files with 195 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts
View File

@ -1,18 +1,20 @@
import { Logger } from '@nestjs/common';
import isEmpty from 'lodash.isempty';
import { Command, CommandRunner, Option } from 'nest-commander';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service';
import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service';
// TODO: implement dry-run
interface RunWorkspaceMigrationsOptions {
workspaceId?: string;
dryRun?: boolean;
force?: boolean;
workspaceId?: string;
}
@Command({
@ -27,6 +29,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
private readonly workspaceHealthService: WorkspaceHealthService,
private readonly dataSourceService: DataSourceService,
private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService,
private readonly workspaceStatusService: WorkspaceStatusService,
) {
super();
}
@ -36,7 +39,19 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
options: RunWorkspaceMigrationsOptions,
): Promise<void> {
// TODO: re-implement load index from workspaceService, this is breaking the logger
const workspaceIds = options.workspaceId ? [options.workspaceId] : [];
let workspaceIds = options.workspaceId ? [options.workspaceId] : [];
if (isEmpty(workspaceIds)) {
const activeWorkspaceIds =
await this.workspaceStatusService.getActiveWorkspaceIds();
workspaceIds = activeWorkspaceIds;
this.logger.log(
`Attempting to sync ${activeWorkspaceIds.length} workspaces.`,
);
}
const errorsDuringSync: string[] = [];
for (const workspaceId of workspaceIds) {
try {
@ -57,7 +72,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
'Please use `workspace:health` command to check issues and fix them before running this command.',
);
return;
continue;
}
this.logger.warn(
@ -75,34 +90,52 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
);
}
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
workspaceId,
);
const { storage, workspaceMigrations } =
await this.workspaceSyncMetadataService.synchronize(
{
try {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
workspaceId,
dataSourceId: dataSourceMetadata.id,
},
{ applyChanges: !options.dryRun },
);
const { storage, workspaceMigrations } =
await this.workspaceSyncMetadataService.synchronize(
{
workspaceId,
dataSourceId: dataSourceMetadata.id,
},
{ applyChanges: !options.dryRun },
);
if (options.dryRun) {
await this.syncWorkspaceLoggerService.saveLogs(
workspaceId,
storage,
workspaceMigrations,
);
}
} catch (error) {
errorsDuringSync.push(
`Failed to synchronize workspace ${workspaceId}: ${error.message}`,
);
if (options.dryRun) {
await this.syncWorkspaceLoggerService.saveLogs(
workspaceId,
storage,
workspaceMigrations,
);
continue;
}
}
this.logger.log(
`Finished synchronizing all active workspaces (${
workspaceIds.length
} workspaces). ${
errorsDuringSync.length > 0
? 'Errors during sync:\n' + errorsDuringSync.join('.\n')
: ''
}`,
);
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: true,
required: false,
})
parseWorkspaceId(value: string): string {
return value;

13
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts
View File

@ -1,12 +1,15 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { AddStandardIdCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/add-standard-id.command';
import { ConvertRecordPositionsToIntegers } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/convert-record-positions-to-integers.command';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
import { SyncWorkspaceMetadataCommand } from './sync-workspace-metadata.command';
@ -19,6 +22,8 @@ import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.ser
WorkspaceModule,
DataSourceModule,
WorkspaceDataSourceModule,
TypeOrmModule.forFeature([Workspace], 'core'),
WorkspaceStatusModule,
],
providers: [
SyncWorkspaceMetadataCommand,

12
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts
View File

@ -5,18 +5,18 @@ import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
import { workspaceSyncMetadataFactories } from 'src/engine/workspace-manager/workspace-sync-metadata/factories';
import { workspaceSyncMetadataComparators } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators';
import { workspaceSyncMetadataFactories } from 'src/engine/workspace-manager/workspace-sync-metadata/factories';
import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service';
import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service';
import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service';
import { WorkspaceSyncObjectMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service';
import { WorkspaceSyncRelationMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-relation-metadata.service';
import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service';
import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service';
import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
@Module({
imports: [