admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add pg-boss worker poc (#2991)

Browse Source 
* Add pg-boss worker poc

* add Example job

* add retry limit

* rename MessageQueue
This commit is contained in:
Weiko
2023-12-14 18:57:25 +01:00
committed by GitHub
parent 468744298b
commit 36164ab59b
18 changed files with 196 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
packages/twenty-server/src/integrations/message-queue/drivers/bullmq.driver.ts
View File

@ -2,25 +2,25 @@ import { Queue, QueueOptions, Worker } from 'bullmq';
import { QueueJobOptions } from 'src/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueues } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueDriver } from './interfaces/message-queue-driver.interface';
export type BullMQDriverOptions = QueueOptions;
export class BullMQDriver implements MessageQueueDriver {
private queueMap: Record<MessageQueues, Queue> = {} as Record<
MessageQueues,
private queueMap: Record<MessageQueue, Queue> = {} as Record<
MessageQueue,
Queue
>;
private workerMap: Record<MessageQueues, Worker> = {} as Record<
MessageQueues,
private workerMap: Record<MessageQueue, Worker> = {} as Record<
MessageQueue,
Worker
>;
constructor(private options: BullMQDriverOptions) {}
register(queueName: MessageQueues): void {
register(queueName: MessageQueue): void {
this.queueMap[queueName] = new Queue(queueName, this.options);
}
@ -35,7 +35,7 @@ export class BullMQDriver implements MessageQueueDriver {
}
async work<T>(
queueName: MessageQueues,
queueName: MessageQueue,
handler: ({ data, id }: { data: T; id: string }) => Promise<void>,
) {
const worker = new Worker(queueName, async (job) => {
@ -46,7 +46,8 @@ export class BullMQDriver implements MessageQueueDriver {
}
async add<T>(
queueName: MessageQueues,
queueName: MessageQueue,
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void> {
@ -55,7 +56,8 @@ export class BullMQDriver implements MessageQueueDriver {
`Queue ${queueName} is not registered, make sure you have added it as a queue provider`,
);
}
await this.queueMap[queueName].add(options?.id || '', data, {
await this.queueMap[queueName].add(jobName, data, {
jobId: options?.id,
priority: options?.priority,
});
}

1
packages/twenty-server/src/integrations/message-queue/drivers/interfaces/job-options.interface.ts
View File

@ -1,4 +1,5 @@
export interface QueueJobOptions {
id?: string;
priority?: number;
retryLimit?: number;
}

9
packages/twenty-server/src/integrations/message-queue/drivers/interfaces/message-queue-driver.interface.ts
View File

@ -1,17 +1,18 @@
import { QueueJobOptions } from 'src/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueues } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
export interface MessageQueueDriver {
add<T>(
queueName: MessageQueues,
queueName: MessageQueue,
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void>;
work<T>(
queueName: string,
queueName: MessageQueue,
handler: ({ data, id }: { data: T; id: string }) => Promise<void> | void,
);
stop?(): Promise<void>;
register?(queueName: MessageQueues): void;
register?(queueName: MessageQueue): void;
}

13
packages/twenty-server/src/integrations/message-queue/drivers/pg-boss.driver.ts
View File

@ -2,6 +2,8 @@ import PgBoss from 'pg-boss';
import { QueueJobOptions } from 'src/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueDriver } from './interfaces/message-queue-driver.interface';
export type PgBossDriverOptions = PgBoss.ConstructorOptions;
@ -25,14 +27,19 @@ export class PgBossDriver implements MessageQueueDriver {
queueName: string,
handler: ({ data, id }: { data: T; id: string }) => Promise<void>,
) {
return this.pgBoss.work(queueName, handler);
return this.pgBoss.work(`${queueName}.*`, handler);
}
async add<T>(
queueName: string,
queueName: MessageQueue,
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void> {
await this.pgBoss.send(queueName, data as object, options ? options : {});
await this.pgBoss.send(
`${queueName}.${jobName}`,
data as object,
options ?? {},
);
}
}

7
packages/twenty-server/src/integrations/message-queue/interfaces/message-queue-job.interface.ts Normal file
View File

@ -0,0 +1,7 @@
export interface MessageQueueJob<T extends MessageQueueJobData> {
handle(data: T): Promise<void> | void;
}
export interface MessageQueueJobData {
[key: string]: any;
}

3
packages/twenty-server/src/integrations/message-queue/message-queue.constants.ts
View File

@ -1,5 +1,6 @@
export const QUEUE_DRIVER = Symbol('QUEUE_DRIVER');
export enum MessageQueues {
export enum MessageQueue {
taskAssignedQueue = 'task-assigned-queue',
messagingQueue = 'messaging-queue',
}

15
packages/twenty-server/src/integrations/message-queue/message-queue.module.ts
View File

@ -8,7 +8,7 @@ import {
} from 'src/integrations/message-queue/interfaces';
import {
QUEUE_DRIVER,
MessageQueues,
MessageQueue,
} from 'src/integrations/message-queue/message-queue.constants';
import { PgBossDriver } from 'src/integrations/message-queue/drivers/pg-boss.driver';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
@ -18,16 +18,13 @@ import { BullMQDriver } from 'src/integrations/message-queue/drivers/bullmq.driv
export class MessageQueueModule {
static forRoot(options: MessageQueueModuleAsyncOptions): DynamicModule {
const providers = [
{
provide: MessageQueues.taskAssignedQueue,
...Object.values(MessageQueue).map((queue) => ({
provide: queue,
useFactory: (driver: MessageQueueDriver) => {
return new MessageQueueService(
driver,
MessageQueues.taskAssignedQueue,
);
return new MessageQueueService(driver, queue);
},
inject: [QUEUE_DRIVER],
},
})),
{
provide: QUEUE_DRIVER,
useFactory: async (...args: any[]) => {
@ -51,7 +48,7 @@ export class MessageQueueModule {
module: MessageQueueModule,
imports: options.imports || [],
providers,
exports: [MessageQueues.taskAssignedQueue],
exports: [MessageQueue.taskAssignedQueue, MessageQueue.messagingQueue],
};
}
}

10
packages/twenty-server/src/integrations/message-queue/services/message-queue-task-assigned.service.spec.ts
View File

@ -4,7 +4,7 @@ import { MessageQueueDriver } from 'src/integrations/message-queue/drivers/inter
import {
QUEUE_DRIVER,
MessageQueues,
MessageQueue,
} from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
@ -15,11 +15,11 @@ describe('MessageQueueTaskAssigned queue', () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
{
provide: MessageQueues.taskAssignedQueue,
provide: MessageQueue.taskAssignedQueue,
useFactory: (driver: MessageQueueDriver) => {
return new MessageQueueService(
driver,
MessageQueues.taskAssignedQueue,
MessageQueue.taskAssignedQueue,
);
},
inject: [QUEUE_DRIVER],
@ -31,7 +31,7 @@ describe('MessageQueueTaskAssigned queue', () => {
],
}).compile();
service = module.get<MessageQueueService>(MessageQueues.taskAssignedQueue);
service = module.get<MessageQueueService>(MessageQueue.taskAssignedQueue);
});
it('should be defined', () => {
@ -40,7 +40,7 @@ describe('MessageQueueTaskAssigned queue', () => {
it('should contain the topic and driver', () => {
expect(service).toEqual({
driver: {},
queueName: MessageQueues.taskAssignedQueue,
queueName: MessageQueue.taskAssignedQueue,
});
});
});

15
packages/twenty-server/src/integrations/message-queue/services/message-queue.service.ts
View File

@ -2,9 +2,10 @@ import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
import { QueueJobOptions } from 'src/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueueDriver } from 'src/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
import { MessageQueueJobData } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import {
MessageQueues,
MessageQueue,
QUEUE_DRIVER,
} from 'src/integrations/message-queue/message-queue.constants';
@ -12,7 +13,7 @@ import {
export class MessageQueueService implements OnModuleDestroy {
constructor(
@Inject(QUEUE_DRIVER) protected driver: MessageQueueDriver,
protected queueName: MessageQueues,
protected queueName: MessageQueue,
) {
if (typeof this.driver.register === 'function') {
this.driver.register(queueName);
@ -25,11 +26,15 @@ export class MessageQueueService implements OnModuleDestroy {
}
}
add<T>(data: T, options?: QueueJobOptions): Promise<void> {
return this.driver.add(this.queueName, data, options);
add<T extends MessageQueueJobData>(
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void> {
return this.driver.add(this.queueName, jobName, data, options);
}
work<T>(
work<T extends MessageQueueJobData>(
handler: ({ data, id }: { data: T; id: string }) => Promise<void> | void,
) {
return this.driver.work(this.queueName, handler);