Improve performance twenty orm (#6691)

## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions

View File

@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace], 'core'),
WorkspaceCacheStorageModule,
],
exports: [WorkspaceMetadataVersionService],
providers: [WorkspaceMetadataVersionService],
})
export class WorkspaceMetadataVersionModule {}

View File

@ -0,0 +1,79 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceMetadataVersionService {
logger = new Logger(WorkspaceMetadataVersionService.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
async flushCacheIfMetadataVersionIsOutdated(
workspaceId: string,
): Promise<void> {
const currentVersion =
(await this.workspaceCacheStorageService.getMetadataVersion(
workspaceId,
)) ?? 1;
let latestVersion = await this.getMetadataVersion(workspaceId);
if (latestVersion === undefined || currentVersion !== latestVersion) {
this.logger.log(
`Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`,
);
await this.workspaceCacheStorageService.flush(workspaceId);
latestVersion = await this.incrementMetadataVersion(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
latestVersion,
);
}
}
async incrementMetadataVersion(workspaceId: string): Promise<number> {
const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0;
const newMetadataVersion = metadataVersion + 1;
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: newMetadataVersion },
);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
newMetadataVersion,
);
return newMetadataVersion;
}
async getMetadataVersion(workspaceId: string): Promise<number | undefined> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
return workspace?.metadataVersion;
}
async resetMetadataVersion(workspaceId: string): Promise<void> {
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: 1 },
);
await this.workspaceCacheStorageService.flush(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1);
}
}