[backend] use cache storage service for workspace schema cache (#4342)

* [backend] add cache storage module

* update docs

* update default TTL to a week

* Use cacheStorageService for WorkspaceSchemaCache

* remove memory storage module

* revert pattern

* remove logs

packages/twenty-server/src/integrations/cache-storage/cache-storage.service.ts

@@ -18,4 +18,8 @@ export class CacheStorageService {
   async set<T>(key: string, value: T, ttl?: number) {
     return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
   }
+
+  async del(key: string) {
+    return this.cacheManager.del(`${this.namespace}:${key}`);
+  }
 }

packages/twenty-server/src/integrations/cache-storage/types/cache-storage-namespace.enum.ts

@@ -1,3 +1,4 @@
 export enum CacheStorageNamespace {
   Messaging = 'messaging',
+  WorkspaceSchema = 'workspaceSchema',
 }

packages/twenty-server/src/integrations/memory-storage/decorators/inject-memory-storage.decorator.ts

@@ -1,9 +0,0 @@
-import { Inject } from '@nestjs/common';
-
-import { createMemoryStorageInjectionToken } from 'src/integrations/memory-storage/memory-storage.util';
-
-export const InjectMemoryStorage = (identifier: string) => {
-  const injectionToken = createMemoryStorageInjectionToken(identifier);
-
-  return Inject(injectionToken);
-};

packages/twenty-server/src/integrations/memory-storage/drivers/interfaces/memory-storage-driver.interface.ts

@@ -1,5 +0,0 @@
-export interface MemoryStorageDriver<T> {
-  read(params: { key: string }): Promise<T | null>;
-  write(params: { key: string; data: T }): Promise<void>;
-  delete(params: { key: string }): Promise<void>;
-}

packages/twenty-server/src/integrations/memory-storage/drivers/local.driver.ts

@@ -1,62 +0,0 @@
-import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
-
-import { MemoryStorageDriver } from './interfaces/memory-storage-driver.interface';
-
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface LocalMemoryDriverOptions {}
-
-export class LocalMemoryDriver<T> implements MemoryStorageDriver<T> {
-  private identifier: string;
-  private options: LocalMemoryDriverOptions;
-  private serializer: MemoryStorageSerializer<T>;
-  private storage: Map<string, string> = new Map();
-
-  constructor(
-    identifier: string,
-    options: LocalMemoryDriverOptions,
-    serializer: MemoryStorageSerializer<T>,
-  ) {
-    this.identifier = identifier;
-    this.options = options;
-    this.serializer = serializer;
-  }
-
-  async write(params: { key: string; data: T }): Promise<void> {
-    const compositeKey = this.generateCompositeKey(params.key);
-    const serializedData = this.serializer.serialize(params.data);
-
-    this.storage.set(compositeKey, serializedData);
-  }
-
-  async read(params: { key: string }): Promise<T | null> {
-    const compositeKey = this.generateCompositeKey(params.key);
-
-    if (!this.storage.has(compositeKey)) {
-      return null;
-    }
-
-    const data = this.storage.get(compositeKey);
-
-    if (!data) {
-      return null;
-    }
-
-    const deserializeData = this.serializer.deserialize(data);
-
-    return deserializeData;
-  }
-
-  async delete(params: { key: string }): Promise<void> {
-    const compositeKey = this.generateCompositeKey(params.key);
-
-    if (!this.storage.has(compositeKey)) {
-      return;
-    }
-
-    this.storage.delete(compositeKey);
-  }
-
-  private generateCompositeKey(key: string): string {
-    return `${this.identifier}:${key}`;
-  }
-}

packages/twenty-server/src/integrations/memory-storage/interfaces/index.ts

@@ -1 +0,0 @@
-export * from './memory-storage.interface';

packages/twenty-server/src/integrations/memory-storage/interfaces/memory-storage.interface.ts

@@ -1,32 +0,0 @@
-import { FactoryProvider, ModuleMetadata } from '@nestjs/common';
-
-import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
-
-import { LocalMemoryDriverOptions } from 'src/integrations/memory-storage/drivers/local.driver';
-
-export enum MemoryStorageDriverType {
-  Local = 'local',
-}
-
-export interface LocalMemoryDriverFactoryOptions {
-  type: MemoryStorageDriverType.Local;
-  options: LocalMemoryDriverOptions;
-}
-
-interface MemoryStorageModuleBaseOptions {
-  identifier: string;
-  serializer?: MemoryStorageSerializer<any>;
-}
-
-export type MemoryStorageModuleOptions = MemoryStorageModuleBaseOptions &
-  LocalMemoryDriverFactoryOptions;
-
-export type MemoryStorageModuleAsyncOptions = {
-  identifier: string;
-  useFactory: (
-    ...args: any[]
-  ) =>
-    | Omit<MemoryStorageModuleOptions, 'identifier'>
-    | Promise<Omit<MemoryStorageModuleOptions, 'identifier'>>;
-} & Pick<ModuleMetadata, 'imports'> &
-  Pick<FactoryProvider, 'inject'>;

packages/twenty-server/src/integrations/memory-storage/memory-storage.constants.ts

@@ -1 +0,0 @@
-export const MEMORY_STORAGE_SERVICE = 'MEMORY_STORAGE_SERVICE';

packages/twenty-server/src/integrations/memory-storage/memory-storage.module.ts

@@ -1,72 +0,0 @@
-import { DynamicModule, Global } from '@nestjs/common';
-
-import { MemoryStorageDefaultSerializer } from 'src/integrations/memory-storage/serializers/default.serializer';
-import { createMemoryStorageInjectionToken } from 'src/integrations/memory-storage/memory-storage.util';
-
-import {
-  MemoryStorageDriverType,
-  MemoryStorageModuleAsyncOptions,
-  MemoryStorageModuleOptions,
-} from './interfaces';
-
-import { LocalMemoryDriver } from './drivers/local.driver';
-
-@Global()
-export class MemoryStorageModule {
-  static forRoot(options: MemoryStorageModuleOptions): DynamicModule {
-    // Dynamic injection token to allow multiple instances of the same driver
-    const injectionToken = createMemoryStorageInjectionToken(
-      options.identifier,
-    );
-    const provider = {
-      provide: injectionToken,
-      useValue: this.createStorageDriver(options),
-    };
-
-    return {
-      module: MemoryStorageModule,
-      providers: [provider],
-      exports: [provider],
-    };
-  }
-
-  static forRootAsync(options: MemoryStorageModuleAsyncOptions): DynamicModule {
-    // Dynamic injection token to allow multiple instances of the same driver
-    const injectionToken = createMemoryStorageInjectionToken(
-      options.identifier,
-    );
-    const provider = {
-      provide: injectionToken,
-      useFactory: async (...args: any[]) => {
-        const config = await options.useFactory(...args);
-
-        return this.createStorageDriver({
-          identifier: options.identifier,
-          ...config,
-        });
-      },
-      inject: options.inject || [],
-    };
-
-    return {
-      module: MemoryStorageModule,
-      imports: options.imports || [],
-      providers: [provider],
-      exports: [provider],
-    };
-  }
-
-  private static createStorageDriver(options: MemoryStorageModuleOptions) {
-    switch (options.type) {
-      case MemoryStorageDriverType.Local:
-        return new LocalMemoryDriver(
-          options.identifier,
-          options.options,
-          options.serializer ?? new MemoryStorageDefaultSerializer<string>(),
-        );
-      // Future case for Redis or other types
-      default:
-        throw new Error(`Unsupported storage type: ${options.type}`);
-    }
-  }
-}

packages/twenty-server/src/integrations/memory-storage/memory-storage.service.spec.ts

@@ -1,19 +0,0 @@
-import { Test, TestingModule } from '@nestjs/testing';
-
-import { MemoryStorageService } from './memory-storage.service';
-
-describe('MemoryStorageService', () => {
-  let service: MemoryStorageService<any>;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      providers: [MemoryStorageService],
-    }).compile();
-
-    service = module.get<MemoryStorageService<any>>(MemoryStorageService);
-  });
-
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});

packages/twenty-server/src/integrations/memory-storage/memory-storage.service.ts

@@ -1,21 +0,0 @@
-import { MemoryStorageDriver } from 'src/integrations/memory-storage/drivers/interfaces/memory-storage-driver.interface';
-
-export class MemoryStorageService<T> implements MemoryStorageDriver<T> {
-  private driver: MemoryStorageDriver<T>;
-
-  constructor(driver: MemoryStorageDriver<T>) {
-    this.driver = driver;
-  }
-
-  write(params: { key: string; data: T }): Promise<void> {
-    return this.driver.write(params);
-  }
-
-  read(params: { key: string }): Promise<T | null> {
-    return this.driver.read(params);
-  }
-
-  delete(params: { key: string }): Promise<void> {
-    return this.driver.delete(params);
-  }
-}

packages/twenty-server/src/integrations/memory-storage/memory-storage.util.ts

@@ -1,5 +0,0 @@
-import { MEMORY_STORAGE_SERVICE } from 'src/integrations/memory-storage/memory-storage.constants';
-
-export const createMemoryStorageInjectionToken = (identifier: string) => {
-  return `${MEMORY_STORAGE_SERVICE}_${identifier}`;
-};

packages/twenty-server/src/integrations/memory-storage/serializers/default.serializer.ts

@@ -1,17 +0,0 @@
-import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
-
-export class MemoryStorageDefaultSerializer<T>
-  implements MemoryStorageSerializer<T>
-{
-  serialize(item: T): string {
-    if (typeof item !== 'string') {
-      throw new Error('DefaultSerializer can only serialize strings');
-    }
-
-    return item;
-  }
-
-  deserialize(data: string): T {
-    return data as unknown as T;
-  }
-}

packages/twenty-server/src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface.ts

@@ -1,4 +0,0 @@
-export interface MemoryStorageSerializer<T> {
-  serialize(item: T): string;
-  deserialize(data: string): T;
-}

packages/twenty-server/src/integrations/memory-storage/serializers/json.serializer.ts

@@ -1,13 +0,0 @@
-import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
-
-export class MemoryStorageJsonSerializer<T>
-  implements MemoryStorageSerializer<T>
-{
-  serialize(item: T): string {
-    return JSON.stringify(item);
-  }
-
-  deserialize(data: string): T {
-    return JSON.parse(data) as T;
-  }
-}

packages/twenty-server/src/workspace/workspace-schema-storage/workspace-schema-storage.module.ts

@@ -1,40 +1,11 @@
 import { Module } from '@nestjs/common';
 
-import { MemoryStorageDriverType } from 'src/integrations/memory-storage/interfaces';
-import { MemoryStorageModule } from 'src/integrations/memory-storage/memory-storage.module';
-import { MemoryStorageJsonSerializer } from 'src/integrations/memory-storage/serializers/json.serializer';
-import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
 import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
 import { WorkspaceCacheVersionModule } from 'src/metadata/workspace-cache-version/workspace-cache-version.module';
 import { WorkspaceSchemaStorageService } from 'src/workspace/workspace-schema-storage/workspace-schema-storage.service';
 
 @Module({
-  imports: [
+  imports: [ObjectMetadataModule, WorkspaceCacheVersionModule],
-    ObjectMetadataModule,
-    WorkspaceCacheVersionModule,
-    MemoryStorageModule.forRoot({
-      identifier: 'objectMetadataCollection',
-      type: MemoryStorageDriverType.Local,
-      options: {},
-      serializer: new MemoryStorageJsonSerializer<ObjectMetadataEntity[]>(),
-    }),
-    MemoryStorageModule.forRoot({
-      identifier: 'typeDefs',
-      type: MemoryStorageDriverType.Local,
-      options: {},
-    }),
-    MemoryStorageModule.forRoot({
-      identifier: 'usedScalarNames',
-      type: MemoryStorageDriverType.Local,
-      options: {},
-      serializer: new MemoryStorageJsonSerializer<string[]>(),
-    }),
-    MemoryStorageModule.forRoot({
-      identifier: 'cacheVersion',
-      type: MemoryStorageDriverType.Local,
-      options: {},
-    }),
-  ],
   providers: [WorkspaceSchemaStorageService],
   exports: [WorkspaceSchemaStorageService],
 })

packages/twenty-server/src/workspace/workspace-schema-storage/workspace-schema-storage.service.ts

@@ -1,33 +1,25 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 
-import { InjectMemoryStorage } from 'src/integrations/memory-storage/decorators/inject-memory-storage.decorator';
+import { CacheStorageService } from 'src/integrations/cache-storage/cache-storage.service';
-import { MemoryStorageService } from 'src/integrations/memory-storage/memory-storage.service';
+import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
 import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
 import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service';
 
 @Injectable()
 export class WorkspaceSchemaStorageService {
   constructor(
-    @InjectMemoryStorage('objectMetadataCollection')
+    @Inject(CacheStorageNamespace.WorkspaceSchema)
-    private readonly objectMetadataMemoryStorageService: MemoryStorageService<
+    private readonly workspaceSchemaCache: CacheStorageService,
-      ObjectMetadataEntity[]
+
-    >,
-    @InjectMemoryStorage('typeDefs')
-    private readonly typeDefsMemoryStorageService: MemoryStorageService<string>,
-    @InjectMemoryStorage('usedScalarNames')
-    private readonly usedScalarNamesMemoryStorageService: MemoryStorageService<
-      string[]
-    >,
-    @InjectMemoryStorage('cacheVersion')
-    private readonly cacheVersionMemoryStorageService: MemoryStorageService<string>,
     private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
   ) {}
 
   async validateCacheVersion(workspaceId: string): Promise<void> {
     const currentVersion =
-      (await this.cacheVersionMemoryStorageService.read({
+      (await this.workspaceSchemaCache.get<string>(
-        key: workspaceId,
+        `cacheVersion:${workspaceId}`,
-      })) ?? '0';
+      )) ?? '0';
+
     let latestVersion =
       await this.workspaceCacheVersionService.getVersion(workspaceId);
 
@@ -40,63 +32,63 @@ export class WorkspaceSchemaStorageService {
         await this.workspaceCacheVersionService.incrementVersion(workspaceId);
 
       // Update the cache version after invalidation
-      await this.cacheVersionMemoryStorageService.write({
+      await this.workspaceSchemaCache.set<string>(
-        key: workspaceId,
+        `cacheVersion:${workspaceId}`,
-        data: latestVersion,
+        latestVersion,
-      });
+      );
     }
   }
 
-  setObjectMetadata(
+  setObjectMetadataCollection(
     workspaceId: string,
-    objectMetadata: ObjectMetadataEntity[],
+    objectMetadataCollection: ObjectMetadataEntity[],
   ) {
-    return this.objectMetadataMemoryStorageService.write({
+    return this.workspaceSchemaCache.set<ObjectMetadataEntity[]>(
-      key: workspaceId,
+      `objectMetadataCollection:${workspaceId}`,
-      data: objectMetadata,
+      objectMetadataCollection,
-    });
+    );
   }
 
-  getObjectMetadata(
+  getObjectMetadataCollection(
     workspaceId: string,
-  ): Promise<ObjectMetadataEntity[] | null> {
+  ): Promise<ObjectMetadataEntity[] | undefined> {
-    return this.objectMetadataMemoryStorageService.read({
+    return this.workspaceSchemaCache.get<ObjectMetadataEntity[]>(
-      key: workspaceId,
+      `objectMetadataCollection:${workspaceId}`,
-    });
+    );
   }
 
   setTypeDefs(workspaceId: string, typeDefs: string): Promise<void> {
-    return this.typeDefsMemoryStorageService.write({
+    return this.workspaceSchemaCache.set<string>(
-      key: workspaceId,
+      `typeDefs:${workspaceId}`,
-      data: typeDefs,
+      typeDefs,
-    });
+    );
   }
 
-  getTypeDefs(workspaceId: string): Promise<string | null> {
+  getTypeDefs(workspaceId: string): Promise<string | undefined> {
-    return this.typeDefsMemoryStorageService.read({
+    return this.workspaceSchemaCache.get<string>(`typeDefs:${workspaceId}`);
-      key: workspaceId,
-    });
   }
 
   setUsedScalarNames(
     workspaceId: string,
     scalarsUsed: string[],
   ): Promise<void> {
-    return this.usedScalarNamesMemoryStorageService.write({
+    return this.workspaceSchemaCache.set<string[]>(
-      key: workspaceId,
+      `usedScalarNames:${workspaceId}`,
-      data: scalarsUsed,
+      scalarsUsed,
-    });
+    );
   }
 
-  getUsedScalarNames(workspaceId: string): Promise<string[] | null> {
+  getUsedScalarNames(workspaceId: string): Promise<string[] | undefined> {
-    return this.usedScalarNamesMemoryStorageService.read({
+    return this.workspaceSchemaCache.get<string[]>(
-      key: workspaceId,
+      `usedScalarNames:${workspaceId}`,
-    });
+    );
   }
 
   async invalidateCache(workspaceId: string): Promise<void> {
-    await this.objectMetadataMemoryStorageService.delete({ key: workspaceId });
+    await this.workspaceSchemaCache.del(
-    await this.typeDefsMemoryStorageService.delete({ key: workspaceId });
+      `objectMetadataCollection:${workspaceId}`,
-    await this.usedScalarNamesMemoryStorageService.delete({ key: workspaceId });
+    );
+    await this.workspaceSchemaCache.del(`typeDefs:${workspaceId}`);
+    await this.workspaceSchemaCache.del(`usedScalarNames:${workspaceId}`);
   }
 }

packages/twenty-server/src/workspace/workspace.factory.ts

@@ -47,14 +47,16 @@ export class WorkspaceFactory {
 
     // Get object metadata from cache
     let objectMetadataCollection =
-      await this.workspaceSchemaStorageService.getObjectMetadata(workspaceId);
+      await this.workspaceSchemaStorageService.getObjectMetadataCollection(
+        workspaceId,
+      );
 
     // If object metadata is not cached, get it from the database
     if (!objectMetadataCollection) {
       objectMetadataCollection =
         await this.objectMetadataService.findManyWithinWorkspace(workspaceId);
 
-      await this.workspaceSchemaStorageService.setObjectMetadata(
+      await this.workspaceSchemaStorageService.setObjectMetadataCollection(
         workspaceId,
         objectMetadataCollection,
       );