mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[messaing] improve messaging import (#4650)

Browse Source 
* [messaging] improve full-sync fetching strategy

* fix

* rebase

* fix

* fix

* fix rebase

* fix

* fix

* fix

* fix

* fix

* remove deletion

* fix setPop with memory storage

* fix pgBoss and remove unnecessary job

* fix throw

* fix

* add timeout to ongoing sync
This commit is contained in:
Weiko
2024-03-27 12:44:03 +01:00
committed by GitHub
parent 5c0b65eecb
commit 5c40e3608b
48 changed files with 1728 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
packages/twenty-server/src/engine/integrations/message-queue/decorators/message-queue.decorator.ts Normal file
View File

@ -0,0 +1,7 @@
import { Inject } from '@nestjs/common';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
export const InjectMessageQueue = (messageQueueName: MessageQueue) => {
return Inject(messageQueueName);
};

12
packages/twenty-server/src/engine/integrations/message-queue/drivers/bullmq.driver.ts
View File

@ -1,6 +1,9 @@
import { Queue, QueueOptions, Worker } from 'bullmq';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
@ -53,8 +56,7 @@ export class BullMQDriver implements MessageQueueDriver {
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
): Promise<void> {
if (!this.queueMap[queueName]) {
throw new Error(
@ -64,9 +66,7 @@ export class BullMQDriver implements MessageQueueDriver {
const queueOptions = {
jobId: options?.id,
priority: options?.priority,
repeat: {
pattern,
},
repeat: options?.repeat,
};
await this.queueMap[queueName].add(jobName, data, queueOptions);

8
packages/twenty-server/src/engine/integrations/message-queue/drivers/interfaces/job-options.interface.ts
View File

@ -3,3 +3,11 @@ export interface QueueJobOptions {
priority?: number;
retryLimit?: number;
}
export interface QueueCronJobOptions extends QueueJobOptions {
repeat?: {
every?: number;
pattern?: string;
limit?: number;
};
}

8
packages/twenty-server/src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface.ts
View File

@ -1,4 +1,7 @@
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
@ -18,8 +21,7 @@ export interface MessageQueueDriver {
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
);
removeCron(queueName: MessageQueue, jobName: string, pattern?: string);
stop?(): Promise<void>;

14
packages/twenty-server/src/engine/integrations/message-queue/drivers/pg-boss.driver.ts
View File

@ -1,6 +1,9 @@
import PgBoss from 'pg-boss';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
@ -8,6 +11,8 @@ import { MessageQueueDriver } from './interfaces/message-queue-driver.interface'
export type PgBossDriverOptions = PgBoss.ConstructorOptions;
const DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED = '*/10 * * * *';
export class PgBossDriver implements MessageQueueDriver {
private pgBoss: PgBoss;
@ -34,16 +39,15 @@ export class PgBossDriver implements MessageQueueDriver {
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
): Promise<void> {
await this.pgBoss.schedule(
`${queueName}.${jobName}`,
pattern,
options?.repeat?.pattern ??
DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED,
data as object,
options
? {
...options,
singletonKey: options?.id,
}
: {},

3
packages/twenty-server/src/engine/integrations/message-queue/drivers/sync.driver.ts
View File

@ -33,9 +33,8 @@ export class SyncDriver implements MessageQueueDriver {
_queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
): Promise<void> {
this.logger.log(`Running '${pattern}' cron job with SyncDriver`);
this.logger.log(`Running cron job with SyncDriver`);
const jobClassName = getJobClassName(jobName);
const job: MessageQueueCronJobData<MessageQueueJobData | undefined> =

23
packages/twenty-server/src/engine/integrations/message-queue/jobs.module.ts
View File

@ -44,9 +44,15 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { MessageParticipantObjectMetadata } from 'src/modules/messaging/standard-objects/message-participant.object-metadata';
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module';
import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module';
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/commands/crons/fetch-all-messages-from-cache.cron-job';
import { GmailFullSyncV2Job } from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
import { GmailPartialSyncV2Job } from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job';
import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module';
@Module({
imports: [
@ -78,6 +84,9 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
MessageChannelObjectMetadata,
EventObjectMetadata,
]),
GmailFullSynV2Module,
GmailFetchMessageContentFromCacheModule,
GmailPartialSyncV2Module,
],
providers: [
{
@ -142,6 +151,18 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
provide: SaveEventToDbJob.name,
useClass: SaveEventToDbJob,
},
{
provide: FetchAllMessagesFromCacheCronJob.name,
useClass: FetchAllMessagesFromCacheCronJob,
},
{
provide: GmailFullSyncV2Job.name,
useClass: GmailFullSyncV2Job,
},
{
provide: GmailPartialSyncV2Job.name,
useClass: GmailPartialSyncV2Job,
},
],
})
export class JobsModule {

10
packages/twenty-server/src/engine/integrations/message-queue/services/message-queue.service.ts
View File

@ -1,6 +1,9 @@
import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
@ -37,10 +40,9 @@ export class MessageQueueService implements OnModuleDestroy {
addCron<T extends MessageQueueJobData | undefined>(
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
): Promise<void> {
return this.driver.addCron(this.queueName, jobName, data, pattern, options);
return this.driver.addCron(this.queueName, jobName, data, options);
}
removeCron(jobName: string, pattern: string): Promise<void> {