[messaing] improve messaging import (#4650)

* [messaging] improve full-sync fetching strategy

* fix

* rebase

* fix

* fix

* fix rebase

* fix

* fix

* fix

* fix

* fix

* remove deletion

* fix setPop with memory storage

* fix pgBoss and remove unnecessary job

* fix throw

* fix

* add timeout to ongoing sync

packages/twenty-server/src/engine/integrations/cache-storage/__tests__/cache-storage.service.spec.ts

@@ -1,78 +0,0 @@
-import { Cache } from '@nestjs/cache-manager';
-
-import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
-import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
-
-const cacheStorageNamespace = CacheStorageNamespace.Messaging;
-
-describe('CacheStorageService', () => {
-  let cacheStorageService: CacheStorageService;
-  let cacheManagerMock: Partial<Cache>;
-
-  beforeEach(() => {
-    cacheManagerMock = {
-      get: jest.fn(),
-      set: jest.fn(),
-    };
-
-    cacheStorageService = new CacheStorageService(
-      cacheManagerMock as Cache,
-      cacheStorageNamespace,
-    );
-  });
-
-  afterEach(() => {
-    jest.clearAllMocks();
-  });
-
-  describe('get', () => {
-    it('should call cacheManager.get with the correct namespaced key', async () => {
-      const key = 'testKey';
-      const namespacedKey = `${cacheStorageNamespace}:${key}`;
-
-      await cacheStorageService.get(key);
-
-      expect(cacheManagerMock.get).toHaveBeenCalledWith(namespacedKey);
-    });
-
-    it('should return the value returned by cacheManager.get', async () => {
-      const key = 'testKey';
-      const value = 'testValue';
-
-      jest.spyOn(cacheManagerMock, 'get').mockResolvedValue(value);
-
-      const result = await cacheStorageService.get(key);
-
-      expect(result).toBe(value);
-    });
-  });
-
-  describe('set', () => {
-    it('should call cacheManager.set with the correct namespaced key, value, and optional ttl', async () => {
-      const key = 'testKey';
-      const value = 'testValue';
-      const ttl = 60;
-      const namespacedKey = `${cacheStorageNamespace}:${key}`;
-
-      await cacheStorageService.set(key, value, ttl);
-
-      expect(cacheManagerMock.set).toHaveBeenCalledWith(
-        namespacedKey,
-        value,
-        ttl,
-      );
-    });
-
-    it('should not throw if cacheManager.set resolves successfully', async () => {
-      const key = 'testKey';
-      const value = 'testValue';
-      const ttl = 60;
-
-      jest.spyOn(cacheManagerMock, 'set').mockResolvedValue(undefined);
-
-      await expect(
-        cacheStorageService.set(key, value, ttl),
-      ).resolves.not.toThrow();
-    });
-  });
-});

packages/twenty-server/src/engine/integrations/cache-storage/cache-storage.service.ts

@@ -1,25 +1,67 @@
 import { Inject, Injectable } from '@nestjs/common';
 import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
 
+import { RedisCache } from 'cache-manager-redis-yet';
+
 import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
 
 @Injectable()
 export class CacheStorageService {
   constructor(
     @Inject(CACHE_MANAGER)
-    private readonly cacheManager: Cache,
+    private readonly cache: Cache,
     private readonly namespace: CacheStorageNamespace,
   ) {}
 
   async get<T>(key: string): Promise<T | undefined> {
-    return this.cacheManager.get(`${this.namespace}:${key}`);
+    return this.cache.get(`${this.namespace}:${key}`);
   }
 
   async set<T>(key: string, value: T, ttl?: number) {
-    return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
+    return this.cache.set(`${this.namespace}:${key}`, value, ttl);
   }
 
   async del(key: string) {
-    return this.cacheManager.del(`${this.namespace}:${key}`);
+    return this.cache.del(`${this.namespace}:${key}`);
+  }
+
+  async setAdd(key: string, value: string[]) {
+    if (value.length === 0) {
+      return;
+    }
+    if (this.isRedisCache()) {
+      return (this.cache as RedisCache).store.client.sAdd(
+        `${this.namespace}:${key}`,
+        value,
+      );
+    }
+    this.get(key).then((res: string[]) => {
+      if (res) {
+        this.set(key, [...res, ...value]);
+      } else {
+        this.set(key, value);
+      }
+    });
+  }
+
+  async setPop(key: string, size: number = 1) {
+    if (this.isRedisCache()) {
+      return (this.cache as RedisCache).store.client.sPop(
+        `${this.namespace}:${key}`,
+        size,
+      );
+    }
+
+    return this.get(key).then((res: string[]) => {
+      if (res) {
+        this.set(key, res.slice(0, -size));
+      }
+
+      return res;
+    });
+  }
+
+  private isRedisCache() {
+    return (this.cache.store as any)?.name === 'redis';
   }
 }

packages/twenty-server/src/engine/integrations/cache-storage/decorators/cache-storage.decorator.ts

@@ -0,0 +1,9 @@
+import { Inject } from '@nestjs/common';
+
+import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
+
+export const InjectCacheStorage = (
+  cacheStorageNamespace: CacheStorageNamespace,
+) => {
+  return Inject(cacheStorageNamespace);
+};