admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor metadata caching (#7011)

Browse Source 
This PR introduces the following changes:
- add the metadataVersion to all our metadata cache keys to ease
troubleshooting:
<img width="1146" alt="image"
src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12">
- introduce a cache recompute lock to avoid overloading the database to
recompute the cache many time
This commit is contained in:
Charles Bochet
2024-09-12 15:57:30 +02:00
committed by Charles Bochet
parent 9b46e8c663
commit 3c4168759a
32 changed files with 420 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception.ts Normal file
View File

@ -0,0 +1,12 @@
import { CustomException } from 'src/utils/custom-exception';
export class WorkspaceMetadataVersionException extends CustomException {
code: WorkspaceMetadataVersionExceptionCode;
constructor(message: string, code: WorkspaceMetadataVersionExceptionCode) {
super(message, code);
}
}
export enum WorkspaceMetadataVersionExceptionCode {
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
}

52
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts Normal file
View File

@ -0,0 +1,52 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
import {
WorkspaceMetadataVersionException,
WorkspaceMetadataVersionExceptionCode,
} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
@Injectable()
export class WorkspaceMetadataVersionService {
logger = new Logger(WorkspaceMetadataCacheService.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {}
async incrementMetadataVersion(workspaceId: string): Promise<void> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
const metadataVersion = workspace?.metadataVersion;
if (metadataVersion === undefined) {
throw new WorkspaceMetadataVersionException(
'Metadata version not found',
WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
);
}
const newMetadataVersion = metadataVersion + 1;
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: newMetadataVersion },
);
await this.workspaceMetadataCacheService.recomputeMetadataCache(
workspaceId,
);
await this.twentyORMGlobalManager.loadDataSourceForWorkspace(workspaceId);
}
}

4
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts
View File

@ -2,13 +2,15 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace], 'core'),
WorkspaceCacheStorageModule,
WorkspaceMetadataCacheModule,
],
exports: [WorkspaceMetadataVersionService],
providers: [WorkspaceMetadataVersionService],

81
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts
View File

@ -1,81 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceMetadataVersionService {
logger = new Logger(WorkspaceMetadataVersionService.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
async flushCacheIfMetadataVersionIsOutdated(
workspaceId: string,
): Promise<void> {
const currentVersion =
(await this.workspaceCacheStorageService.getMetadataVersion(
workspaceId,
)) ?? 1;
let latestVersion = await this.getMetadataVersion(workspaceId);
if (latestVersion === undefined || currentVersion !== latestVersion) {
this.logger.log(
`Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`,
);
await this.workspaceCacheStorageService.flush(workspaceId);
latestVersion = await this.incrementMetadataVersion(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
latestVersion,
);
}
}
async incrementMetadataVersion(workspaceId: string): Promise<number> {
const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0;
const newMetadataVersion = metadataVersion + 1;
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: newMetadataVersion },
);
await this.workspaceCacheStorageService.flush(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
newMetadataVersion,
);
return newMetadataVersion;
}
async getMetadataVersion(workspaceId: string): Promise<number | undefined> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
return workspace?.metadataVersion;
}
async resetMetadataVersion(workspaceId: string): Promise<void> {
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: 1 },
);
await this.workspaceCacheStorageService.flush(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1);
}
}