feat: Enhancements to MessageQueue Module with Decorators (#5657)

### Overview

This PR introduces significant enhancements to the MessageQueue module
by integrating `@Processor`, `@Process`, and `@InjectMessageQueue`
decorators. These changes streamline the process of defining and
managing queue processors and job handlers, and also allow for
request-scoped handlers, improving compatibility with services that rely
on scoped providers like TwentyORM repositories.

### Key Features

1. **Decorator-based Job Handling**: Use `@Processor` and `@Process`
decorators to define job handlers declaratively.
2. **Request Scope Support**: Job handlers can be scoped per request,
enhancing integration with request-scoped services.

### Usage

#### Defining Processors and Job Handlers

The `@Processor` decorator is used to define a class that processes jobs
for a specific queue. The `@Process` decorator is applied to methods
within this class to define specific job handlers.

##### Example 1: Specific Job Handlers

```typescript
import { Processor, Process, InjectMessageQueue } from 'src/engine/integrations/message-queue';

@Processor('taskQueue')
export class TaskProcessor {

  @Process('taskA')
  async handleTaskA(job: { id: string, data: any }) {
    console.log(`Handling task A with data:`, job.data);
    // Logic for task A
  }

  @Process('taskB')
  async handleTaskB(job: { id: string, data: any }) {
    console.log(`Handling task B with data:`, job.data);
    // Logic for task B
  }
}
```

In the example above, `TaskProcessor` is responsible for processing jobs
in the `taskQueue`. The `handleTaskA` method will only be called for
jobs with the name `taskA`, while `handleTaskB` will be called for
`taskB` jobs.

##### Example 2: General Job Handler

```typescript
import { Processor, Process, InjectMessageQueue } from 'src/engine/integrations/message-queue';

@Processor('generalQueue')
export class GeneralProcessor {

  @Process()
  async handleAnyJob(job: { id: string, name: string, data: any }) {
    console.log(`Handling job ${job.name} with data:`, job.data);
    // Logic for any job
  }
}
```

In this example, `GeneralProcessor` handles all jobs in the
`generalQueue`, regardless of the job name. The `handleAnyJob` method
will be invoked for every job added to the `generalQueue`.

#### Adding Jobs to a Queue

You can use the `@InjectMessageQueue` decorator to inject a queue into a
service and add jobs to it.

##### Example:

```typescript
import { Injectable } from '@nestjs/common';
import { InjectMessageQueue, MessageQueue } from 'src/engine/integrations/message-queue';

@Injectable()
export class TaskService {
  constructor(
    @InjectMessageQueue('taskQueue') private readonly taskQueue: MessageQueue,
  ) {}

  async addTaskA(data: any) {
    await this.taskQueue.add('taskA', data);
  }

  async addTaskB(data: any) {
    await this.taskQueue.add('taskB', data);
  }
}
```

In this example, `TaskService` adds jobs to the `taskQueue`. The
`addTaskA` and `addTaskB` methods add jobs named `taskA` and `taskB`,
respectively, to the queue.

#### Using Scoped Job Handlers

To utilize request-scoped job handlers, specify the scope in the
`@Processor` decorator. This is particularly useful for services that
use scoped repositories like those in TwentyORM.

##### Example:

```typescript
import { Processor, Process, InjectMessageQueue, Scope } from 'src/engine/integrations/message-queue';

@Processor({ name: 'scopedQueue', scope: Scope.REQUEST })
export class ScopedTaskProcessor {

  @Process('scopedTask')
  async handleScopedTask(job: { id: string, data: any }) {
    console.log(`Handling scoped task with data:`, job.data);
    // Logic for scoped task, which might use request-scoped services
  }
}
```

Here, the `ScopedTaskProcessor` is associated with `scopedQueue` and
operates with request scope. This setup is essential when the job
handler relies on services that need to be instantiated per request,
such as scoped repositories.

### Migration Notes

- **Decorators**: Refactor job handlers to use `@Processor` and
`@Process` decorators.
- **Request Scope**: Utilize the scope option in `@Processor` if your
job handlers depend on request-scoped services.

Fix #5628

---------

Co-authored-by: Weiko <corentin@twenty.com>

packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job.ts

@@ -1,6 +1,5 @@
-import { Inject, Injectable, Logger } from '@nestjs/common';
+import { Logger } from '@nestjs/common';
 
-import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
 import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
 
 import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
@@ -11,6 +10,9 @@ import {
   CallWebhookJob,
   CallWebhookJobData,
 } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job';
+import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
+import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
+import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
 
 export enum CallWebhookJobsJobOperation {
   create = 'create',
@@ -25,19 +27,18 @@ export type CallWebhookJobsJobData = {
   operation: CallWebhookJobsJobOperation;
 };
 
-@Injectable()
-export class CallWebhookJobsJob
-  implements MessageQueueJob<CallWebhookJobsJobData>
-{
+@Processor(MessageQueue.webhookQueue)
+export class CallWebhookJobsJob {
   private readonly logger = new Logger(CallWebhookJobsJob.name);
 
   constructor(
     private readonly workspaceDataSourceService: WorkspaceDataSourceService,
     private readonly dataSourceService: DataSourceService,
-    @Inject(MessageQueue.webhookQueue)
+    @InjectMessageQueue(MessageQueue.webhookQueue)
     private readonly messageQueueService: MessageQueueService,
   ) {}
 
+  @Process(CallWebhookJobsJob.name)
   async handle(data: CallWebhookJobsJobData): Promise<void> {
     const dataSourceMetadata =
       await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(

packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job.ts

@@ -1,7 +1,9 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Logger } from '@nestjs/common';
 import { HttpService } from '@nestjs/axios';
 
-import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
+import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
+import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
+import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
 
 export type CallWebhookJobData = {
   targetUrl: string;
@@ -13,12 +15,13 @@ export type CallWebhookJobData = {
   record: any;
 };
 
-@Injectable()
-export class CallWebhookJob implements MessageQueueJob<CallWebhookJobData> {
+@Processor(MessageQueue.webhookQueue)
+export class CallWebhookJob {
   private readonly logger = new Logger(CallWebhookJob.name);
 
   constructor(private readonly httpService: HttpService) {}
 
+  @Process(CallWebhookJob.name)
   async handle(data: CallWebhookJobData): Promise<void> {
     try {
       await this.httpService.axiosRef.post(data.targetUrl, data);

packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/record-position-backfill.job.ts

@@ -1,22 +1,20 @@
-import { Injectable } from '@nestjs/common';
-
-import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
-
 import { RecordPositionBackfillService } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-service';
+import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
+import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
+import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
 
 export type RecordPositionBackfillJobData = {
   workspaceId: string;
   dryRun: boolean;
 };
 
-@Injectable()
-export class RecordPositionBackfillJob
-  implements MessageQueueJob<RecordPositionBackfillJobData>
-{
+@Processor(MessageQueue.recordPositionBackfillQueue)
+export class RecordPositionBackfillJob {
   constructor(
     private readonly recordPositionBackfillService: RecordPositionBackfillService,
   ) {}
 
+  @Process(RecordPositionBackfillJob.name)
   async handle(data: RecordPositionBackfillJobData): Promise<void> {
     this.recordPositionBackfillService.backfill(data.workspaceId, data.dryRun);
   }

packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/workspace-query-runner-job.module.ts

@@ -15,19 +15,6 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
     RecordPositionBackfillModule,
     HttpModule,
   ],
-  providers: [
-    {
-      provide: CallWebhookJobsJob.name,
-      useClass: CallWebhookJobsJob,
-    },
-    {
-      provide: CallWebhookJob.name,
-      useClass: CallWebhookJob,
-    },
-    {
-      provide: RecordPositionBackfillJob.name,
-      useClass: RecordPositionBackfillJob,
-    },
-  ],
+  providers: [CallWebhookJobsJob, CallWebhookJob, RecordPositionBackfillJob],
 })
 export class WorkspaceQueryRunnerJobModule {}