Add a limit to workflow queue per workspace (#12908)

- new status `ENQUEUED` added. With a command to backfill
- counter in cache per workspace, managed by a new service
[workflow-run-queue.workspace-service.ts](https://github.com/twentyhq/twenty/compare/tt-improve-workflow-run-queueing?expand=1#diff-1e2de2a48cd482a3bd7e8dedf1150a19d0b200afbd9282181a24ecddddb56927)
- cron added that will run every minute to look for not started
workflows

Here is the new flow:
- When executing a workflow, we check if the queue is not full. If not,
run is created as `ENQUEUED` and the run workflow job is triggered as
usual. If full, create the run as NOT_STARTED and do not trigger the job
- Cron will look for NOT_STARTED workflows and queue some if there is
some place again
- Only MANUAL and Form submit skip the queue limit

packages/twenty-server/src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run.command.ts

@@ -0,0 +1,112 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { Repository } from 'typeorm';
+
+import {
+  ActiveOrSuspendedWorkspacesMigrationCommandRunner,
+  RunOnWorkspaceArgs,
+} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
+import { WORKFLOW_RUN_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
+import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
+
+@Command({
+  name: 'upgrade:1-2:add-enqueued-status-to-workflow-run',
+  description: 'Add enqueued status to workflow run',
+})
+export class AddEnqueuedStatusToWorkflowRunCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(Workspace, 'core')
+    protected readonly workspaceRepository: Repository<Workspace>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    @InjectRepository(FieldMetadataEntity, 'core')
+    private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
+    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    this.logger.log(
+      `Adding enqueued status to workflow run for workspace ${workspaceId}`,
+    );
+
+    const workflowRunStatusFieldMetadata =
+      await this.fieldMetadataRepository.findOne({
+        where: {
+          standardId: WORKFLOW_RUN_STANDARD_FIELD_IDS.status,
+        },
+      });
+
+    if (!workflowRunStatusFieldMetadata) {
+      this.logger.error(
+        `Workflow run status field metadata not found for workspace ${workspaceId}`,
+      );
+
+      return;
+    }
+
+    const workflowRunStatusFieldMetadataOptions =
+      workflowRunStatusFieldMetadata.options;
+
+    // check if enqueued status is already in the field metadata options
+    if (
+      workflowRunStatusFieldMetadataOptions.some(
+        (option) => option.value === WorkflowRunStatus.ENQUEUED,
+      )
+    ) {
+      this.logger.log(
+        `Workflow run status field metadata options already contain enqueued status for workspace ${workspaceId}`,
+      );
+    } else if (options.dryRun) {
+      this.logger.log(
+        `Would add enqueued status to workflow run status field metadata for workspace ${workspaceId}`,
+      );
+    } else {
+      workflowRunStatusFieldMetadataOptions.push({
+        value: WorkflowRunStatus.ENQUEUED,
+        label: 'Enqueued',
+        position: 4,
+        color: 'blue',
+      });
+
+      await this.fieldMetadataRepository.save(workflowRunStatusFieldMetadata);
+
+      this.logger.log(
+        `Enqueued status added to workflow run status field metadata for workspace ${workspaceId}`,
+      );
+    }
+
+    const schemaName =
+      this.workspaceDataSourceService.getSchemaName(workspaceId);
+
+    const mainDataSource =
+      await this.workspaceDataSourceService.connectToMainDataSource();
+
+    if (options.dryRun) {
+      this.logger.log(
+        `Would try to add enqueued status to workflow run status enum for workspace ${workspaceId}`,
+      );
+    } else {
+      try {
+        await mainDataSource.query(
+          `ALTER TYPE ${schemaName}."workflowRun_status_enum" ADD VALUE 'ENQUEUED'`,
+        );
+        this.logger.log(
+          `Enqueued status added to workflow run status enum for workspace ${workspaceId}`,
+        );
+      } catch (error) {
+        this.logger.error(
+          `Error adding enqueued status to workflow run status enum for workspace ${workspaceId}: ${error}`,
+        );
+      }
+    }
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version-command/1-2/1-2-upgrade-version-command.module.ts

@@ -0,0 +1,17 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { AddEnqueuedStatusToWorkflowRunCommand } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run.command';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([Workspace, FieldMetadataEntity], 'core'),
+    WorkspaceDataSourceModule,
+  ],
+  providers: [AddEnqueuedStatusToWorkflowRunCommand],
+  exports: [AddEnqueuedStatusToWorkflowRunCommand],
+})
+export class V1_2_UpgradeVersionCommandModule {}

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts

@@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { V0_54_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-54/0-54-upgrade-version-command.module';
 import { V0_55_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module';
 import { V1_1_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-1/1-1-upgrade-version-command.module';
+import { V1_2_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-2/1-2-upgrade-version-command.module';
 import {
   DatabaseMigrationService,
   UpgradeCommand,
@@ -17,6 +18,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
     V0_54_UpgradeVersionCommandModule,
     V0_55_UpgradeVersionCommandModule,
     V1_1_UpgradeVersionCommandModule,
+    V1_2_UpgradeVersionCommandModule,
     WorkspaceSyncMetadataModule,
   ],
   providers: [DatabaseMigrationService, UpgradeCommand],

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts

@@ -23,6 +23,7 @@ import { MigrateDefaultAvatarUrlToUserWorkspaceCommand } from 'src/database/comm
 import { DeduplicateIndexedFieldsCommand } from 'src/database/commands/upgrade-version-command/0-55/0-55-deduplicate-indexed-fields.command';
 import { FixSchemaArrayTypeCommand } from 'src/database/commands/upgrade-version-command/1-1/1-1-fix-schema-array-type.command';
 import { FixUpdateStandardFieldsIsLabelSyncedWithName } from 'src/database/commands/upgrade-version-command/1-1/1-1-fix-update-standard-field-is-label-synced-with-name.command';
+import { AddEnqueuedStatusToWorkflowRunCommand } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run.command';
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
@@ -141,6 +142,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
     // 1.1 Commands
     protected readonly fixSchemaArrayTypeCommand: FixSchemaArrayTypeCommand,
     protected readonly fixUpdateStandardFieldsIsLabelSyncedWithNameCommand: FixUpdateStandardFieldsIsLabelSyncedWithName,
+
+    // 1.2 Commands
+    protected readonly addEnqueuedStatusToWorkflowRunCommand: AddEnqueuedStatusToWorkflowRunCommand,
   ) {
     super(
       workspaceRepository,
@@ -189,6 +193,11 @@ export class UpgradeCommand extends UpgradeCommandRunner {
       afterSyncMetadata: [],
     };
 
+    const commands_120: VersionCommands = {
+      beforeSyncMetadata: [this.addEnqueuedStatusToWorkflowRunCommand],
+      afterSyncMetadata: [],
+    };
+
     this.allCommands = {
       '0.53.0': commands_053,
       '0.54.0': commands_054,
@@ -196,6 +205,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
       '0.60.0': commands_060,
       '1.0.0': commands_100,
       '1.1.0': commands_110,
+      '1.2.0': commands_120,
     };
   }