Optimize sync, reset, seed commands to flush cache and to use less memory (#7034)

In this PR:
- removing ugprade-0.24 commands as we are releasing 0.30
- introducing cache:flush command
- refactoring upgrade command and sync-metadata command to use the
ActiveWorkspacesCommand so they consistently run on all workspaces or
selected workspaces

Fixes:
- clear localStorage on sign out
- fix missing workspaceMember in verify resolver
- do not throw on datasource already destroyed exception which can
happen with race condition when several resolvers are resolving in
parallel

packages/twenty-server/src/engine/workspace-manager/workspace-manager.module.ts

@@ -5,7 +5,6 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
 import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
 import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
 import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
-import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
 import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
 
 import { WorkspaceManagerService } from './workspace-manager.service';
@@ -18,7 +17,6 @@ import { WorkspaceManagerService } from './workspace-manager.service';
     DataSourceModule,
     WorkspaceSyncMetadataModule,
     WorkspaceHealthModule,
-    WorkspaceStatusModule,
   ],
   exports: [WorkspaceManagerService],
   providers: [WorkspaceManagerService],

packages/twenty-server/src/engine/workspace-manager/workspace-status/services/workspace-status.service.ts

@@ -1,72 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
-
-import { Any, Repository } from 'typeorm';
-
-import {
-  BillingSubscription,
-  SubscriptionStatus,
-} from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
-import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
-import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
-import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
-import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
-
-@Injectable()
-export class WorkspaceStatusService {
-  constructor(
-    private readonly environmentService: EnvironmentService,
-    @InjectRepository(Workspace, 'core')
-    private readonly workspaceRepository: Repository<Workspace>,
-    @InjectRepository(BillingSubscription, 'core')
-    private readonly billingSubscriptionRepository: Repository<BillingSubscription>,
-    @InjectRepository(FeatureFlagEntity, 'core')
-    private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
-  ) {}
-
-  async getActiveWorkspaceIds(): Promise<string[]> {
-    const workspaces = await this.workspaceRepository.find();
-    const workspaceIds = workspaces.map((workspace) => workspace.id);
-
-    if (!this.environmentService.get('IS_BILLING_ENABLED')) {
-      return workspaceIds;
-    }
-
-    const billingSubscriptionForWorkspaces =
-      await this.billingSubscriptionRepository.find({
-        where: {
-          workspaceId: Any(workspaceIds),
-          status: Any([
-            SubscriptionStatus.PastDue,
-            SubscriptionStatus.Active,
-            SubscriptionStatus.Trialing,
-          ]),
-        },
-      });
-
-    const workspaceIdsWithActiveSubscription =
-      billingSubscriptionForWorkspaces.map(
-        (billingSubscription) => billingSubscription.workspaceId,
-      );
-
-    const freeAccessEnabledFeatureFlagForWorkspace =
-      await this.featureFlagRepository.find({
-        where: {
-          workspaceId: Any(workspaceIds),
-          key: FeatureFlagKey.IsFreeAccessEnabled,
-          value: true,
-        },
-      });
-
-    const workspaceIdsWithFreeAccessEnabled =
-      freeAccessEnabledFeatureFlagForWorkspace.map(
-        (featureFlag) => featureFlag.workspaceId,
-      );
-
-    return workspaceIds.filter(
-      (workspaceId) =>
-        workspaceIdsWithActiveSubscription.includes(workspaceId) ||
-        workspaceIdsWithFreeAccessEnabled.includes(workspaceId),
-    );
-  }
-}

packages/twenty-server/src/engine/workspace-manager/workspace-status/workspace-manager.module.ts

@@ -1,21 +0,0 @@
-import { Module } from '@nestjs/common';
-import { TypeOrmModule } from '@nestjs/typeorm';
-
-import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
-import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
-import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
-import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module';
-import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service';
-
-@Module({
-  imports: [
-    EnvironmentModule,
-    TypeOrmModule.forFeature(
-      [Workspace, BillingSubscription, FeatureFlagEntity],
-      'core',
-    ),
-  ],
-  exports: [WorkspaceStatusService],
-  providers: [WorkspaceStatusService],
-})
-export class WorkspaceStatusModule {}

packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts

@@ -1,11 +1,12 @@
-import { Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
 
-import isEmpty from 'lodash.isempty';
-import { Command, CommandRunner, Option } from 'nest-commander';
+import { Command, Option } from 'nest-commander';
+import { Repository } from 'typeorm';
 
+import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
-import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service';
 import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
 
 import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service';
@@ -21,35 +22,24 @@ interface RunWorkspaceMigrationsOptions {
   name: 'workspace:sync-metadata',
   description: 'Sync metadata',
 })
-export class SyncWorkspaceMetadataCommand extends CommandRunner {
-  private readonly logger = new Logger(SyncWorkspaceMetadataCommand.name);
-
+export class SyncWorkspaceMetadataCommand extends ActiveWorkspacesCommandRunner {
   constructor(
+    @InjectRepository(Workspace, 'core')
+    protected readonly workspaceRepository: Repository<Workspace>,
     private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService,
     private readonly workspaceHealthService: WorkspaceHealthService,
     private readonly dataSourceService: DataSourceService,
     private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService,
-    private readonly workspaceStatusService: WorkspaceStatusService,
   ) {
-    super();
+    super(workspaceRepository);
   }
 
-  async run(
+  async executeActiveWorkspacesCommand(
     _passedParam: string[],
     options: RunWorkspaceMigrationsOptions,
+    workspaceIds: string[],
   ): Promise<void> {
-    // TODO: re-implement load index from workspaceService, this is breaking the logger
-    let workspaceIds = options.workspaceId ? [options.workspaceId] : [];
-
-    if (isEmpty(workspaceIds)) {
-      const activeWorkspaceIds =
-        await this.workspaceStatusService.getActiveWorkspaceIds();
-
-      workspaceIds = activeWorkspaceIds;
-      this.logger.log(
-        `Attempting to sync ${activeWorkspaceIds.length} workspaces.`,
-      );
-    }
+    this.logger.log(`Attempting to sync ${workspaceIds.length} workspaces.`);
 
     let count = 1;
 
@@ -138,15 +128,6 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
     );
   }
 
-  @Option({
-    flags: '-w, --workspace-id [workspace_id]',
-    description: 'workspace id',
-    required: false,
-  })
-  parseWorkspaceId(value: string): string {
-    return value;
-  }
-
   @Option({
     flags: '-d, --dry-run',
     description: 'Dry run without applying changes',

packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts

@@ -6,7 +6,6 @@ import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.mod
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
 import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
-import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
 import { ConvertRecordPositionsToIntegers } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/convert-record-positions-to-integers.command';
 import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
 
@@ -22,7 +21,6 @@ import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.ser
     DataSourceModule,
     WorkspaceDataSourceModule,
     TypeOrmModule.forFeature([Workspace], 'core'),
-    WorkspaceStatusModule,
   ],
   providers: [
     SyncWorkspaceMetadataCommand,