[backend] add cache storage module (#4320)

* [backend] add cache storage module

* update docs

* update default TTL to a week

packages/twenty-docs/docs/start/self-hosting/self-hosting.mdx

@@ -41,6 +41,8 @@ import TabItem from '@theme/TabItem';
     ['FRONT_BASE_URL', 'http://localhost:3001', 'Url to the hosted frontend'],
     ['SERVER_URL', 'http://localhost:3000', 'Url to the hosted server'],
     ['PORT', '3000', 'Port'],
+    ['CACHE_STORAGE_TYPE', 'memory', 'Cache type (memory, redis...)'],
+    ['CACHE_STORAGE_TTL', '3600 * 24 * 7', 'Cache TTL in seconds']
     ]}></OptionTable>
 
 ### Security

packages/twenty-server/package.json

@@ -32,8 +32,11 @@
   },
   "dependencies": {
     "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch",
+    "@nestjs/cache-manager": "^2.2.1",
     "@nestjs/graphql": "patch:@nestjs/graphql@12.0.8#./patches/@nestjs+graphql+12.0.8.patch",
     "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch",
+    "cache-manager": "^5.4.0",
+    "cache-manager-redis-yet": "^4.1.2",
     "class-validator": "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch",
     "graphql-middleware": "^6.1.35",
     "passport": "^0.7.0",

packages/twenty-server/src/integrations/cache-storage/__tests__/cache-storage.service.spec.ts

@@ -0,0 +1,78 @@
+import { Cache } from '@nestjs/cache-manager';
+
+import { CacheStorageService } from 'src/integrations/cache-storage/cache-storage.service';
+import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
+
+const cacheStorageNamespace = CacheStorageNamespace.Messaging;
+
+describe('CacheStorageService', () => {
+  let cacheStorageService: CacheStorageService;
+  let cacheManagerMock: Partial<Cache>;
+
+  beforeEach(() => {
+    cacheManagerMock = {
+      get: jest.fn(),
+      set: jest.fn(),
+    };
+
+    cacheStorageService = new CacheStorageService(
+      cacheManagerMock as Cache,
+      cacheStorageNamespace,
+    );
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('get', () => {
+    it('should call cacheManager.get with the correct namespaced key', async () => {
+      const key = 'testKey';
+      const namespacedKey = `${cacheStorageNamespace}:${key}`;
+
+      await cacheStorageService.get(key);
+
+      expect(cacheManagerMock.get).toHaveBeenCalledWith(namespacedKey);
+    });
+
+    it('should return the value returned by cacheManager.get', async () => {
+      const key = 'testKey';
+      const value = 'testValue';
+
+      jest.spyOn(cacheManagerMock, 'get').mockResolvedValue(value);
+
+      const result = await cacheStorageService.get(key);
+
+      expect(result).toBe(value);
+    });
+  });
+
+  describe('set', () => {
+    it('should call cacheManager.set with the correct namespaced key, value, and optional ttl', async () => {
+      const key = 'testKey';
+      const value = 'testValue';
+      const ttl = 60;
+      const namespacedKey = `${cacheStorageNamespace}:${key}`;
+
+      await cacheStorageService.set(key, value, ttl);
+
+      expect(cacheManagerMock.set).toHaveBeenCalledWith(
+        namespacedKey,
+        value,
+        ttl,
+      );
+    });
+
+    it('should not throw if cacheManager.set resolves successfully', async () => {
+      const key = 'testKey';
+      const value = 'testValue';
+      const ttl = 60;
+
+      jest.spyOn(cacheManagerMock, 'set').mockResolvedValue(undefined);
+
+      await expect(
+        cacheStorageService.set(key, value, ttl),
+      ).resolves.not.toThrow();
+    });
+  });
+});

packages/twenty-server/src/integrations/cache-storage/cache-storage.module-factory.ts

@@ -0,0 +1,46 @@
+import { CacheModuleOptions } from '@nestjs/common';
+
+import { redisStore } from 'cache-manager-redis-yet';
+
+import { CacheStorageType } from 'src/integrations/cache-storage/types/cache-storage-type.enum';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+
+export const cacheStorageModuleFactory = (
+  environmentService: EnvironmentService,
+): CacheModuleOptions => {
+  const cacheStorageType = environmentService.getCacheStorageType();
+  const cacheStorageTtl = environmentService.getCacheStorageTtl();
+  const cacheModuleOptions: CacheModuleOptions = {
+    isGlobal: true,
+    ttl: cacheStorageTtl * 1000,
+  };
+
+  switch (cacheStorageType) {
+    case CacheStorageType.Memory: {
+      return cacheModuleOptions;
+    }
+    case CacheStorageType.Redis: {
+      const host = environmentService.getRedisHost();
+      const port = environmentService.getRedisPort();
+
+      if (!(host && port)) {
+        throw new Error(
+          `${cacheStorageType} cache storage requires host: ${host} and port: ${port} to be defined, check your .env file`,
+        );
+      }
+
+      return {
+        ...cacheModuleOptions,
+        store: redisStore,
+        socket: {
+          host,
+          port,
+        },
+      };
+    }
+    default:
+      throw new Error(
+        `Invalid cache-storage (${cacheStorageType}), check your .env file`,
+      );
+  }
+};

packages/twenty-server/src/integrations/cache-storage/cache-storage.module.ts

@@ -0,0 +1,31 @@
+import { Module, Global } from '@nestjs/common';
+import { CacheModule, CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
+import { ConfigModule } from '@nestjs/config';
+
+import { CacheStorageService } from 'src/integrations/cache-storage/cache-storage.service';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+import { cacheStorageModuleFactory } from 'src/integrations/cache-storage/cache-storage.module-factory';
+import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
+
+@Global()
+@Module({
+  imports: [
+    CacheModule.registerAsync({
+      isGlobal: true,
+      imports: [ConfigModule],
+      useFactory: cacheStorageModuleFactory,
+      inject: [EnvironmentService],
+    }),
+  ],
+  providers: [
+    ...Object.values(CacheStorageNamespace).map((cacheStorageNamespace) => ({
+      provide: cacheStorageNamespace,
+      useFactory: (cacheManager: Cache) => {
+        return new CacheStorageService(cacheManager, cacheStorageNamespace);
+      },
+      inject: [CACHE_MANAGER],
+    })),
+  ],
+  exports: [...Object.values(CacheStorageNamespace)],
+})
+export class CacheStorageModule {}

packages/twenty-server/src/integrations/cache-storage/cache-storage.service.ts

@@ -0,0 +1,21 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
+
+import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
+
+@Injectable()
+export class CacheStorageService {
+  constructor(
+    @Inject(CACHE_MANAGER)
+    private readonly cacheManager: Cache,
+    private readonly namespace: CacheStorageNamespace,
+  ) {}
+
+  async get<T>(key: string): Promise<T | undefined> {
+    return this.cacheManager.get(`${this.namespace}:${key}`);
+  }
+
+  async set<T>(key: string, value: T, ttl?: number) {
+    return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
+  }
+}

packages/twenty-server/src/integrations/cache-storage/types/cache-storage-namespace.enum.ts

@@ -0,0 +1,3 @@
+export enum CacheStorageNamespace {
+  Messaging = 'messaging',
+}

packages/twenty-server/src/integrations/cache-storage/types/cache-storage-type.enum.ts

@@ -0,0 +1,4 @@
+export enum CacheStorageType {
+  Memory = 'memory',
+  Redis = 'redis',
+}

packages/twenty-server/src/integrations/environment/environment.service.ts

@@ -328,4 +328,12 @@ export class EnvironmentService {
       this.configService.get<number>('MUTATION_MAXIMUM_RECORD_AFFECTED') ?? 100
     );
   }
+
+  getCacheStorageType(): string {
+    return this.configService.get<string>('CACHE_STORAGE_TYPE') ?? 'memory';
+  }
+
+  getCacheStorageTtl(): number {
+    return this.configService.get<number>('CACHE_STORAGE_TTL') ?? 3600 * 24 * 7;
+  }
 }

packages/twenty-server/src/integrations/integrations.module.ts

@@ -9,6 +9,7 @@ import { loggerModuleFactory } from 'src/integrations/logger/logger.module-facto
 import { messageQueueModuleFactory } from 'src/integrations/message-queue/message-queue.module-factory';
 import { EmailModule } from 'src/integrations/email/email.module';
 import { emailModuleFactory } from 'src/integrations/email/email.module-factory';
+import { CacheStorageModule } from 'src/integrations/cache-storage/cache-storage.module';
 
 import { EnvironmentModule } from './environment/environment.module';
 import { EnvironmentService } from './environment/environment.service';
@@ -40,6 +41,7 @@ import { MessageQueueModule } from './message-queue/message-queue.module';
       inject: [EnvironmentService],
     }),
     EventEmitterModule.forRoot(),
+    CacheStorageModule,
   ],
   exports: [],
   providers: [],

packages/twenty-server/tsconfig.json

@@ -31,5 +31,6 @@
   "ts-node": {
     "files": true,
     "require": ["tsconfig-paths/register"]
-  }
+  },
+  "exclude": ["dist"]
 }