[messaing] improve messaging import (#4650)

* [messaging] improve full-sync fetching strategy

* fix

* rebase

* fix

* fix

* fix rebase

* fix

* fix

* fix

* fix

* fix

* remove deletion

* fix setPop with memory storage

* fix pgBoss and remove unnecessary job

* fix throw

* fix

* add timeout to ongoing sync

packages/twenty-server/src/engine/integrations/message-queue/drivers/bullmq.driver.ts

@@ -1,6 +1,9 @@
 import { Queue, QueueOptions, Worker } from 'bullmq';
 
-import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
+import {
+  QueueCronJobOptions,
+  QueueJobOptions,
+} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
 
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
 
@@ -53,8 +56,7 @@ export class BullMQDriver implements MessageQueueDriver {
     queueName: MessageQueue,
     jobName: string,
     data: T,
-    pattern: string,
-    options?: QueueJobOptions,
+    options?: QueueCronJobOptions,
   ): Promise<void> {
     if (!this.queueMap[queueName]) {
       throw new Error(
@@ -64,9 +66,7 @@ export class BullMQDriver implements MessageQueueDriver {
     const queueOptions = {
       jobId: options?.id,
       priority: options?.priority,
-      repeat: {
-        pattern,
-      },
+      repeat: options?.repeat,
     };
 
     await this.queueMap[queueName].add(jobName, data, queueOptions);

packages/twenty-server/src/engine/integrations/message-queue/drivers/interfaces/job-options.interface.ts

@@ -3,3 +3,11 @@ export interface QueueJobOptions {
   priority?: number;
   retryLimit?: number;
 }
+
+export interface QueueCronJobOptions extends QueueJobOptions {
+  repeat?: {
+    every?: number;
+    pattern?: string;
+    limit?: number;
+  };
+}

packages/twenty-server/src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface.ts

@@ -1,4 +1,7 @@
-import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
+import {
+  QueueCronJobOptions,
+  QueueJobOptions,
+} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
 import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
 
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
@@ -18,8 +21,7 @@ export interface MessageQueueDriver {
     queueName: MessageQueue,
     jobName: string,
     data: T,
-    pattern: string,
-    options?: QueueJobOptions,
+    options?: QueueCronJobOptions,
   );
   removeCron(queueName: MessageQueue, jobName: string, pattern?: string);
   stop?(): Promise<void>;

packages/twenty-server/src/engine/integrations/message-queue/drivers/pg-boss.driver.ts

@@ -1,6 +1,9 @@
 import PgBoss from 'pg-boss';
 
-import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
+import {
+  QueueCronJobOptions,
+  QueueJobOptions,
+} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
 
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
 
@@ -8,6 +11,8 @@ import { MessageQueueDriver } from './interfaces/message-queue-driver.interface'
 
 export type PgBossDriverOptions = PgBoss.ConstructorOptions;
 
+const DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED = '*/10 * * * *';
+
 export class PgBossDriver implements MessageQueueDriver {
   private pgBoss: PgBoss;
 
@@ -34,16 +39,15 @@ export class PgBossDriver implements MessageQueueDriver {
     queueName: MessageQueue,
     jobName: string,
     data: T,
-    pattern: string,
-    options?: QueueJobOptions,
+    options?: QueueCronJobOptions,
   ): Promise<void> {
     await this.pgBoss.schedule(
       `${queueName}.${jobName}`,
-      pattern,
+      options?.repeat?.pattern ??
+        DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED,
       data as object,
       options
         ? {
-            ...options,
             singletonKey: options?.id,
           }
         : {},

packages/twenty-server/src/engine/integrations/message-queue/drivers/sync.driver.ts

@@ -33,9 +33,8 @@ export class SyncDriver implements MessageQueueDriver {
     _queueName: MessageQueue,
     jobName: string,
     data: T,
-    pattern: string,
   ): Promise<void> {
-    this.logger.log(`Running '${pattern}' cron job with SyncDriver`);
+    this.logger.log(`Running cron job with SyncDriver`);
 
     const jobClassName = getJobClassName(jobName);
     const job: MessageQueueCronJobData<MessageQueueJobData | undefined> =