admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[backend] use cache storage service for workspace schema cache (#4342)

Browse Source 
* [backend] add cache storage module

* update docs

* update default TTL to a week

* Use cacheStorageService for WorkspaceSchemaCache

* remove memory storage module

* revert pattern

* remove logs
This commit is contained in:
Weiko
2024-03-12 13:51:39 +01:00
committed by GitHub
parent 1b485c2984
commit 0d8e700239
18 changed files with 54 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/twenty-server/src/integrations/cache-storage/cache-storage.service.ts
View File

@ -18,4 +18,8 @@ export class CacheStorageService {
async set<T>(key: string, value: T, ttl?: number) {
return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
}
async del(key: string) {
return this.cacheManager.del(`${this.namespace}:${key}`);
}
}

1
packages/twenty-server/src/integrations/cache-storage/types/cache-storage-namespace.enum.ts
View File

@ -1,3 +1,4 @@
export enum CacheStorageNamespace {
Messaging = 'messaging',
WorkspaceSchema = 'workspaceSchema',
}

9
packages/twenty-server/src/integrations/memory-storage/decorators/inject-memory-storage.decorator.ts
View File

@ -1,9 +0,0 @@
import { Inject } from '@nestjs/common';
import { createMemoryStorageInjectionToken } from 'src/integrations/memory-storage/memory-storage.util';
export const InjectMemoryStorage = (identifier: string) => {
const injectionToken = createMemoryStorageInjectionToken(identifier);
return Inject(injectionToken);
};

5
packages/twenty-server/src/integrations/memory-storage/drivers/interfaces/memory-storage-driver.interface.ts
View File

@ -1,5 +0,0 @@
export interface MemoryStorageDriver<T> {
read(params: { key: string }): Promise<T | null>;
write(params: { key: string; data: T }): Promise<void>;
delete(params: { key: string }): Promise<void>;
}

62
packages/twenty-server/src/integrations/memory-storage/drivers/local.driver.ts
View File

@ -1,62 +0,0 @@
import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
import { MemoryStorageDriver } from './interfaces/memory-storage-driver.interface';
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface LocalMemoryDriverOptions {}
export class LocalMemoryDriver<T> implements MemoryStorageDriver<T> {
private identifier: string;
private options: LocalMemoryDriverOptions;
private serializer: MemoryStorageSerializer<T>;
private storage: Map<string, string> = new Map();
constructor(
identifier: string,
options: LocalMemoryDriverOptions,
serializer: MemoryStorageSerializer<T>,
) {
this.identifier = identifier;
this.options = options;
this.serializer = serializer;
}
async write(params: { key: string; data: T }): Promise<void> {
const compositeKey = this.generateCompositeKey(params.key);
const serializedData = this.serializer.serialize(params.data);
this.storage.set(compositeKey, serializedData);
}
async read(params: { key: string }): Promise<T | null> {
const compositeKey = this.generateCompositeKey(params.key);
if (!this.storage.has(compositeKey)) {
return null;
}
const data = this.storage.get(compositeKey);
if (!data) {
return null;
}
const deserializeData = this.serializer.deserialize(data);
return deserializeData;
}
async delete(params: { key: string }): Promise<void> {
const compositeKey = this.generateCompositeKey(params.key);
if (!this.storage.has(compositeKey)) {
return;
}
this.storage.delete(compositeKey);
}
private generateCompositeKey(key: string): string {
return `${this.identifier}:${key}`;
}
}

1
packages/twenty-server/src/integrations/memory-storage/interfaces/index.ts
View File

@ -1 +0,0 @@
export * from './memory-storage.interface';

32
packages/twenty-server/src/integrations/memory-storage/interfaces/memory-storage.interface.ts
View File

@ -1,32 +0,0 @@
import { FactoryProvider, ModuleMetadata } from '@nestjs/common';
import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
import { LocalMemoryDriverOptions } from 'src/integrations/memory-storage/drivers/local.driver';
export enum MemoryStorageDriverType {
Local = 'local',
}
export interface LocalMemoryDriverFactoryOptions {
type: MemoryStorageDriverType.Local;
options: LocalMemoryDriverOptions;
}
interface MemoryStorageModuleBaseOptions {
identifier: string;
serializer?: MemoryStorageSerializer<any>;
}
export type MemoryStorageModuleOptions = MemoryStorageModuleBaseOptions &
LocalMemoryDriverFactoryOptions;
export type MemoryStorageModuleAsyncOptions = {
identifier: string;
useFactory: (
...args: any[]
) =>
| Omit<MemoryStorageModuleOptions, 'identifier'>
| Promise<Omit<MemoryStorageModuleOptions, 'identifier'>>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;

1
packages/twenty-server/src/integrations/memory-storage/memory-storage.constants.ts
View File

@ -1 +0,0 @@
export const MEMORY_STORAGE_SERVICE = 'MEMORY_STORAGE_SERVICE';

72
packages/twenty-server/src/integrations/memory-storage/memory-storage.module.ts
View File

@ -1,72 +0,0 @@
import { DynamicModule, Global } from '@nestjs/common';
import { MemoryStorageDefaultSerializer } from 'src/integrations/memory-storage/serializers/default.serializer';
import { createMemoryStorageInjectionToken } from 'src/integrations/memory-storage/memory-storage.util';
import {
MemoryStorageDriverType,
MemoryStorageModuleAsyncOptions,
MemoryStorageModuleOptions,
} from './interfaces';
import { LocalMemoryDriver } from './drivers/local.driver';
@Global()
export class MemoryStorageModule {
static forRoot(options: MemoryStorageModuleOptions): DynamicModule {
// Dynamic injection token to allow multiple instances of the same driver
const injectionToken = createMemoryStorageInjectionToken(
options.identifier,
);
const provider = {
provide: injectionToken,
useValue: this.createStorageDriver(options),
};
return {
module: MemoryStorageModule,
providers: [provider],
exports: [provider],
};
}
static forRootAsync(options: MemoryStorageModuleAsyncOptions): DynamicModule {
// Dynamic injection token to allow multiple instances of the same driver
const injectionToken = createMemoryStorageInjectionToken(
options.identifier,
);
const provider = {
provide: injectionToken,
useFactory: async (...args: any[]) => {
const config = await options.useFactory(...args);
return this.createStorageDriver({
identifier: options.identifier,
...config,
});
},
inject: options.inject || [],
};
return {
module: MemoryStorageModule,
imports: options.imports || [],
providers: [provider],
exports: [provider],
};
}
private static createStorageDriver(options: MemoryStorageModuleOptions) {
switch (options.type) {
case MemoryStorageDriverType.Local:
return new LocalMemoryDriver(
options.identifier,
options.options,
options.serializer ?? new MemoryStorageDefaultSerializer<string>(),
);
// Future case for Redis or other types
default:
throw new Error(`Unsupported storage type: ${options.type}`);
}
}
}

19
packages/twenty-server/src/integrations/memory-storage/memory-storage.service.spec.ts
View File

@ -1,19 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { MemoryStorageService } from './memory-storage.service';
describe('MemoryStorageService', () => {
let service: MemoryStorageService<any>;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [MemoryStorageService],
}).compile();
service = module.get<MemoryStorageService<any>>(MemoryStorageService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

21
packages/twenty-server/src/integrations/memory-storage/memory-storage.service.ts
View File

@ -1,21 +0,0 @@
import { MemoryStorageDriver } from 'src/integrations/memory-storage/drivers/interfaces/memory-storage-driver.interface';
export class MemoryStorageService<T> implements MemoryStorageDriver<T> {
private driver: MemoryStorageDriver<T>;
constructor(driver: MemoryStorageDriver<T>) {
this.driver = driver;
}
write(params: { key: string; data: T }): Promise<void> {
return this.driver.write(params);
}
read(params: { key: string }): Promise<T | null> {
return this.driver.read(params);
}
delete(params: { key: string }): Promise<void> {
return this.driver.delete(params);
}
}

5
packages/twenty-server/src/integrations/memory-storage/memory-storage.util.ts
View File

@ -1,5 +0,0 @@
import { MEMORY_STORAGE_SERVICE } from 'src/integrations/memory-storage/memory-storage.constants';
export const createMemoryStorageInjectionToken = (identifier: string) => {
return `${MEMORY_STORAGE_SERVICE}_${identifier}`;
};

17
packages/twenty-server/src/integrations/memory-storage/serializers/default.serializer.ts
View File

@ -1,17 +0,0 @@
import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
export class MemoryStorageDefaultSerializer<T>
implements MemoryStorageSerializer<T>
{
serialize(item: T): string {
if (typeof item !== 'string') {
throw new Error('DefaultSerializer can only serialize strings');
}
return item;
}
deserialize(data: string): T {
return data as unknown as T;
}
}

4
packages/twenty-server/src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface.ts
View File

@ -1,4 +0,0 @@
export interface MemoryStorageSerializer<T> {
serialize(item: T): string;
deserialize(data: string): T;
}

13
packages/twenty-server/src/integrations/memory-storage/serializers/json.serializer.ts
View File

@ -1,13 +0,0 @@
import { MemoryStorageSerializer } from 'src/integrations/memory-storage/serializers/interfaces/memory-storage-serializer.interface';
export class MemoryStorageJsonSerializer<T>
implements MemoryStorageSerializer<T>
{
serialize(item: T): string {
return JSON.stringify(item);
}
deserialize(data: string): T {
return JSON.parse(data) as T;
}
}

31
packages/twenty-server/src/workspace/workspace-schema-storage/workspace-schema-storage.module.ts
View File

@ -1,40 +1,11 @@
import { Module } from '@nestjs/common';
import { MemoryStorageDriverType } from 'src/integrations/memory-storage/interfaces';
import { MemoryStorageModule } from 'src/integrations/memory-storage/memory-storage.module';
import { MemoryStorageJsonSerializer } from 'src/integrations/memory-storage/serializers/json.serializer';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { WorkspaceCacheVersionModule } from 'src/metadata/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceSchemaStorageService } from 'src/workspace/workspace-schema-storage/workspace-schema-storage.service';
@Module({
imports: [
ObjectMetadataModule,
WorkspaceCacheVersionModule,
MemoryStorageModule.forRoot({
identifier: 'objectMetadataCollection',
type: MemoryStorageDriverType.Local,
options: {},
serializer: new MemoryStorageJsonSerializer<ObjectMetadataEntity[]>(),
}),
MemoryStorageModule.forRoot({
identifier: 'typeDefs',
type: MemoryStorageDriverType.Local,
options: {},
}),
MemoryStorageModule.forRoot({
identifier: 'usedScalarNames',
type: MemoryStorageDriverType.Local,
options: {},
serializer: new MemoryStorageJsonSerializer<string[]>(),
}),
MemoryStorageModule.forRoot({
identifier: 'cacheVersion',
type: MemoryStorageDriverType.Local,
options: {},
}),
],
imports: [ObjectMetadataModule, WorkspaceCacheVersionModule],
providers: [WorkspaceSchemaStorageService],
exports: [WorkspaceSchemaStorageService],
})

96
packages/twenty-server/src/workspace/workspace-schema-storage/workspace-schema-storage.service.ts
View File

@ -1,33 +1,25 @@
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { InjectMemoryStorage } from 'src/integrations/memory-storage/decorators/inject-memory-storage.decorator';
import { MemoryStorageService } from 'src/integrations/memory-storage/memory-storage.service';
import { CacheStorageService } from 'src/integrations/cache-storage/cache-storage.service';
import { CacheStorageNamespace } from 'src/integrations/cache-storage/types/cache-storage-namespace.enum';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service';
@Injectable()
export class WorkspaceSchemaStorageService {
constructor(
@InjectMemoryStorage('objectMetadataCollection')
private readonly objectMetadataMemoryStorageService: MemoryStorageService<
ObjectMetadataEntity[]
>,
@InjectMemoryStorage('typeDefs')
private readonly typeDefsMemoryStorageService: MemoryStorageService<string>,
@InjectMemoryStorage('usedScalarNames')
private readonly usedScalarNamesMemoryStorageService: MemoryStorageService<
string[]
>,
@InjectMemoryStorage('cacheVersion')
private readonly cacheVersionMemoryStorageService: MemoryStorageService<string>,
@Inject(CacheStorageNamespace.WorkspaceSchema)
private readonly workspaceSchemaCache: CacheStorageService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
) {}
async validateCacheVersion(workspaceId: string): Promise<void> {
const currentVersion =
(await this.cacheVersionMemoryStorageService.read({
key: workspaceId,
})) ?? '0';
(await this.workspaceSchemaCache.get<string>(
`cacheVersion:${workspaceId}`,
)) ?? '0';
let latestVersion =
await this.workspaceCacheVersionService.getVersion(workspaceId);
@ -40,63 +32,63 @@ export class WorkspaceSchemaStorageService {
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
// Update the cache version after invalidation
await this.cacheVersionMemoryStorageService.write({
key: workspaceId,
data: latestVersion,
});
await this.workspaceSchemaCache.set<string>(
`cacheVersion:${workspaceId}`,
latestVersion,
);
}
}
setObjectMetadata(
setObjectMetadataCollection(
workspaceId: string,
objectMetadata: ObjectMetadataEntity[],
objectMetadataCollection: ObjectMetadataEntity[],
) {
return this.objectMetadataMemoryStorageService.write({
key: workspaceId,
data: objectMetadata,
});
return this.workspaceSchemaCache.set<ObjectMetadataEntity[]>(
`objectMetadataCollection:${workspaceId}`,
objectMetadataCollection,
);
}
getObjectMetadata(
getObjectMetadataCollection(
workspaceId: string,
): Promise<ObjectMetadataEntity[] | null> {
return this.objectMetadataMemoryStorageService.read({
key: workspaceId,
});
): Promise<ObjectMetadataEntity[] | undefined> {
return this.workspaceSchemaCache.get<ObjectMetadataEntity[]>(
`objectMetadataCollection:${workspaceId}`,
);
}
setTypeDefs(workspaceId: string, typeDefs: string): Promise<void> {
return this.typeDefsMemoryStorageService.write({
key: workspaceId,
data: typeDefs,
});
return this.workspaceSchemaCache.set<string>(
`typeDefs:${workspaceId}`,
typeDefs,
);
}
getTypeDefs(workspaceId: string): Promise<string | null> {
return this.typeDefsMemoryStorageService.read({
key: workspaceId,
});
getTypeDefs(workspaceId: string): Promise<string | undefined> {
return this.workspaceSchemaCache.get<string>(`typeDefs:${workspaceId}`);
}
setUsedScalarNames(
workspaceId: string,
scalarsUsed: string[],
): Promise<void> {
return this.usedScalarNamesMemoryStorageService.write({
key: workspaceId,
data: scalarsUsed,
});
return this.workspaceSchemaCache.set<string[]>(
`usedScalarNames:${workspaceId}`,
scalarsUsed,
);
}
getUsedScalarNames(workspaceId: string): Promise<string[] | null> {
return this.usedScalarNamesMemoryStorageService.read({
key: workspaceId,
});
getUsedScalarNames(workspaceId: string): Promise<string[] | undefined> {
return this.workspaceSchemaCache.get<string[]>(
`usedScalarNames:${workspaceId}`,
);
}
async invalidateCache(workspaceId: string): Promise<void> {
await this.objectMetadataMemoryStorageService.delete({ key: workspaceId });
await this.typeDefsMemoryStorageService.delete({ key: workspaceId });
await this.usedScalarNamesMemoryStorageService.delete({ key: workspaceId });
await this.workspaceSchemaCache.del(
`objectMetadataCollection:${workspaceId}`,
);
await this.workspaceSchemaCache.del(`typeDefs:${workspaceId}`);
await this.workspaceSchemaCache.del(`usedScalarNames:${workspaceId}`);
}
}

6
packages/twenty-server/src/workspace/workspace.factory.ts
View File

@ -47,14 +47,16 @@ export class WorkspaceFactory {
// Get object metadata from cache
let objectMetadataCollection =
await this.workspaceSchemaStorageService.getObjectMetadata(workspaceId);
await this.workspaceSchemaStorageService.getObjectMetadataCollection(
workspaceId,
);
// If object metadata is not cached, get it from the database
if (!objectMetadataCollection) {
objectMetadataCollection =
await this.objectMetadataService.findManyWithinWorkspace(workspaceId);
await this.workspaceSchemaStorageService.setObjectMetadata(
await this.workspaceSchemaStorageService.setObjectMetadataCollection(
workspaceId,
objectMetadataCollection,
);