feat: Enhancements to MessageQueue Module with Decorators (#5657)

### Overview

This PR introduces significant enhancements to the MessageQueue module
by integrating `@Processor`, `@Process`, and `@InjectMessageQueue`
decorators. These changes streamline the process of defining and
managing queue processors and job handlers, and also allow for
request-scoped handlers, improving compatibility with services that rely
on scoped providers like TwentyORM repositories.

### Key Features

1. **Decorator-based Job Handling**: Use `@Processor` and `@Process`
decorators to define job handlers declaratively.
2. **Request Scope Support**: Job handlers can be scoped per request,
enhancing integration with request-scoped services.

### Usage

#### Defining Processors and Job Handlers

The `@Processor` decorator is used to define a class that processes jobs
for a specific queue. The `@Process` decorator is applied to methods
within this class to define specific job handlers.

##### Example 1: Specific Job Handlers

```typescript
import { Processor, Process, InjectMessageQueue } from 'src/engine/integrations/message-queue';

@Processor('taskQueue')
export class TaskProcessor {

  @Process('taskA')
  async handleTaskA(job: { id: string, data: any }) {
    console.log(`Handling task A with data:`, job.data);
    // Logic for task A
  }

  @Process('taskB')
  async handleTaskB(job: { id: string, data: any }) {
    console.log(`Handling task B with data:`, job.data);
    // Logic for task B
  }
}
```

In the example above, `TaskProcessor` is responsible for processing jobs
in the `taskQueue`. The `handleTaskA` method will only be called for
jobs with the name `taskA`, while `handleTaskB` will be called for
`taskB` jobs.

##### Example 2: General Job Handler

```typescript
import { Processor, Process, InjectMessageQueue } from 'src/engine/integrations/message-queue';

@Processor('generalQueue')
export class GeneralProcessor {

  @Process()
  async handleAnyJob(job: { id: string, name: string, data: any }) {
    console.log(`Handling job ${job.name} with data:`, job.data);
    // Logic for any job
  }
}
```

In this example, `GeneralProcessor` handles all jobs in the
`generalQueue`, regardless of the job name. The `handleAnyJob` method
will be invoked for every job added to the `generalQueue`.

#### Adding Jobs to a Queue

You can use the `@InjectMessageQueue` decorator to inject a queue into a
service and add jobs to it.

##### Example:

```typescript
import { Injectable } from '@nestjs/common';
import { InjectMessageQueue, MessageQueue } from 'src/engine/integrations/message-queue';

@Injectable()
export class TaskService {
  constructor(
    @InjectMessageQueue('taskQueue') private readonly taskQueue: MessageQueue,
  ) {}

  async addTaskA(data: any) {
    await this.taskQueue.add('taskA', data);
  }

  async addTaskB(data: any) {
    await this.taskQueue.add('taskB', data);
  }
}
```

In this example, `TaskService` adds jobs to the `taskQueue`. The
`addTaskA` and `addTaskB` methods add jobs named `taskA` and `taskB`,
respectively, to the queue.

#### Using Scoped Job Handlers

To utilize request-scoped job handlers, specify the scope in the
`@Processor` decorator. This is particularly useful for services that
use scoped repositories like those in TwentyORM.

##### Example:

```typescript
import { Processor, Process, InjectMessageQueue, Scope } from 'src/engine/integrations/message-queue';

@Processor({ name: 'scopedQueue', scope: Scope.REQUEST })
export class ScopedTaskProcessor {

  @Process('scopedTask')
  async handleScopedTask(job: { id: string, data: any }) {
    console.log(`Handling scoped task with data:`, job.data);
    // Logic for scoped task, which might use request-scoped services
  }
}
```

Here, the `ScopedTaskProcessor` is associated with `scopedQueue` and
operates with request scope. This setup is essential when the job
handler relies on services that need to be instantiated per request,
such as scoped repositories.

### Migration Notes

- **Decorators**: Refactor job handlers to use `@Processor` and
`@Process` decorators.
- **Request Scope**: Utilize the scope option in `@Processor` if your
job handlers depend on request-scoped services.

Fix #5628

---------

Co-authored-by: Weiko <corentin@twenty.com>
This commit is contained in:
Jérémy M
2024-06-17 09:49:37 +02:00
committed by GitHub
parent 605945bd42
commit d99b9d1d6b
92 changed files with 955 additions and 524 deletions

View File

@ -1,7 +1,6 @@
import { Inject } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { CleanInactiveWorkspaceJob } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-inactive-workspace.job';
@ -16,7 +15,7 @@ export type CleanInactiveWorkspacesCommandOptions = {
})
export class CleanInactiveWorkspacesCommand extends CommandRunner {
constructor(
@Inject(MessageQueue.taskAssignedQueue)
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();

View File

@ -1,7 +1,6 @@
import { Inject } from '@nestjs/common';
import { Command, CommandRunner } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { cleanInactiveWorkspaceCronPattern } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-inactive-workspace.cron.pattern';
@ -13,7 +12,7 @@ import { CleanInactiveWorkspaceJob } from 'src/engine/workspace-manager/workspac
})
export class StartCleanInactiveWorkspacesCronCommand extends CommandRunner {
constructor(
@Inject(MessageQueue.cronQueue)
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();

View File

@ -1,7 +1,6 @@
import { Inject } from '@nestjs/common';
import { Command, CommandRunner } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { cleanInactiveWorkspaceCronPattern } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-inactive-workspace.cron.pattern';
@ -13,7 +12,7 @@ import { CleanInactiveWorkspaceJob } from 'src/engine/workspace-manager/workspac
})
export class StopCleanInactiveWorkspacesCronCommand extends CommandRunner {
constructor(
@Inject(MessageQueue.cronQueue)
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();

View File

@ -1,4 +1,4 @@
import { Injectable, Logger } from '@nestjs/common';
import { Logger } from '@nestjs/common';
import { render } from '@react-email/render';
import { In } from 'typeorm';
@ -7,8 +7,6 @@ import {
DeleteInactiveWorkspaceEmail,
} from 'twenty-emails';
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
@ -20,6 +18,9 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { CleanInactiveWorkspacesCommandOptions } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-inactive-workspaces.command';
import { getDryRunLogHeader } from 'src/utils/get-dry-run-log-header';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
const MILLISECONDS_IN_ONE_DAY = 1000 * 3600 * 24;
@ -28,10 +29,8 @@ type WorkspaceToDeleteData = {
daysSinceInactive: number;
};
@Injectable()
export class CleanInactiveWorkspaceJob
implements MessageQueueJob<CleanInactiveWorkspacesCommandOptions>
{
@Processor(MessageQueue.cronQueue)
export class CleanInactiveWorkspaceJob {
private readonly logger = new Logger(CleanInactiveWorkspaceJob.name);
private readonly inactiveDaysBeforeDelete;
private readonly inactiveDaysBeforeEmail;
@ -193,6 +192,7 @@ export class CleanInactiveWorkspaceJob
});
}
@Process(CleanInactiveWorkspaceJob.name)
async handle(data: CleanInactiveWorkspacesCommandOptions): Promise<void> {
const isDryRun = data.dryRun || false;