admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor metadata caching (#7011)

Browse Source 
This PR introduces the following changes:
- add the metadataVersion to all our metadata cache keys to ease
troubleshooting:
<img width="1146" alt="image"
src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12">
- introduce a cache recompute lock to avoid overloading the database to
recompute the cache many time
This commit is contained in:
Charles Bochet
2024-09-12 15:57:30 +02:00
committed by Charles Bochet
parent 9b46e8c663
commit 3c4168759a
32 changed files with 420 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
View File

@ -41,7 +41,7 @@ import { NameTooLongException } from 'src/engine/metadata-modules/utils/exceptio
import { exceedsDatabaseIdentifierMaximumLength } from 'src/engine/metadata-modules/utils/validate-database-identifier-length.utils';
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
import { validateMetadataNameValidityOrThrow as validateFieldNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,

2
packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts
View File

@ -34,7 +34,7 @@ import {
import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete';
import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service';
import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,

2
packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts
View File

@ -21,7 +21,7 @@ import {
import { InvalidStringException } from 'src/engine/metadata-modules/utils/exceptions/invalid-string.exception';
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
import { validateMetadataNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,

2
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts
View File

@ -11,7 +11,7 @@ import {
} from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.exception';
import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
ReferencedTable,

2
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts
View File

@ -36,7 +36,7 @@ import {
mapUdtNameToFieldType,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,

12
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts Normal file
View File

@ -0,0 +1,12 @@
import { CustomException } from 'src/utils/custom-exception';
export class WorkspaceMetadataCacheException extends CustomException {
code: WorkspaceMetadataCacheExceptionCode;
constructor(message: string, code: WorkspaceMetadataCacheExceptionCode) {
super(message, code);
}
}
export enum WorkspaceMetadataCacheExceptionCode {
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
}

112
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts Normal file
View File

@ -0,0 +1,112 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import {
WorkspaceMetadataCacheException,
WorkspaceMetadataCacheExceptionCode,
} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceMetadataCacheService {
logger = new Logger(WorkspaceMetadataCacheService.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
) {}
async recomputeMetadataCache(
workspaceId: string,
force = false,
): Promise<void> {
const currentCacheVersion =
await this.getMetadataVersionFromCache(workspaceId);
const currentDatabaseVersion =
await this.getMetadataVersionFromDatabase(workspaceId);
if (currentDatabaseVersion === undefined) {
throw new WorkspaceMetadataCacheException(
'Metadata version not found in the database',
WorkspaceMetadataCacheExceptionCode.METADATA_VERSION_NOT_FOUND,
);
}
if (!force && currentCacheVersion === currentDatabaseVersion) {
return;
}
const isAlreadyCaching =
await this.workspaceCacheStorageService.getObjectMetadataCollectionOngoingCachingLock(
workspaceId,
currentDatabaseVersion,
);
if (isAlreadyCaching) {
return;
}
if (currentCacheVersion !== undefined) {
this.workspaceCacheStorageService.flush(workspaceId, currentCacheVersion);
}
await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock(
workspaceId,
currentDatabaseVersion,
);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
currentDatabaseVersion,
);
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
currentDatabaseVersion,
freshObjectMetadataCollection,
);
await this.workspaceCacheStorageService.removeObjectMetadataCollectionOngoingCachingLock(
workspaceId,
currentDatabaseVersion,
);
}
private async getMetadataVersionFromDatabase(
workspaceId: string,
): Promise<number | undefined> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
return workspace?.metadataVersion;
}
private async getMetadataVersionFromCache(
workspaceId: string,
): Promise<number | undefined> {
return await this.workspaceCacheStorageService.getMetadataVersion(
workspaceId,
);
}
}

18
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts Normal file
View File

@ -0,0 +1,18 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace], 'core'),
TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
WorkspaceCacheStorageModule,
],
exports: [WorkspaceMetadataCacheService],
providers: [WorkspaceMetadataCacheService],
})
export class WorkspaceMetadataCacheModule {}

12
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception.ts Normal file
View File

@ -0,0 +1,12 @@
import { CustomException } from 'src/utils/custom-exception';
export class WorkspaceMetadataVersionException extends CustomException {
code: WorkspaceMetadataVersionExceptionCode;
constructor(message: string, code: WorkspaceMetadataVersionExceptionCode) {
super(message, code);
}
}
export enum WorkspaceMetadataVersionExceptionCode {
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
}

52
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts Normal file
View File

@ -0,0 +1,52 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
import {
WorkspaceMetadataVersionException,
WorkspaceMetadataVersionExceptionCode,
} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
@Injectable()
export class WorkspaceMetadataVersionService {
logger = new Logger(WorkspaceMetadataCacheService.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {}
async incrementMetadataVersion(workspaceId: string): Promise<void> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
const metadataVersion = workspace?.metadataVersion;
if (metadataVersion === undefined) {
throw new WorkspaceMetadataVersionException(
'Metadata version not found',
WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
);
}
const newMetadataVersion = metadataVersion + 1;
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: newMetadataVersion },
);
await this.workspaceMetadataCacheService.recomputeMetadataCache(
workspaceId,
);
await this.twentyORMGlobalManager.loadDataSourceForWorkspace(workspaceId);
}
}

4
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts
View File

@ -2,13 +2,15 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace], 'core'),
WorkspaceCacheStorageModule,
WorkspaceMetadataCacheModule,
],
exports: [WorkspaceMetadataVersionService],
providers: [WorkspaceMetadataVersionService],

81
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts
View File

@ -1,81 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceMetadataVersionService {
logger = new Logger(WorkspaceMetadataVersionService.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
async flushCacheIfMetadataVersionIsOutdated(
workspaceId: string,
): Promise<void> {
const currentVersion =
(await this.workspaceCacheStorageService.getMetadataVersion(
workspaceId,
)) ?? 1;
let latestVersion = await this.getMetadataVersion(workspaceId);
if (latestVersion === undefined || currentVersion !== latestVersion) {
this.logger.log(
`Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`,
);
await this.workspaceCacheStorageService.flush(workspaceId);
latestVersion = await this.incrementMetadataVersion(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
latestVersion,
);
}
}
async incrementMetadataVersion(workspaceId: string): Promise<number> {
const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0;
const newMetadataVersion = metadataVersion + 1;
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: newMetadataVersion },
);
await this.workspaceCacheStorageService.flush(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
newMetadataVersion,
);
return newMetadataVersion;
}
async getMetadataVersion(workspaceId: string): Promise<number | undefined> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
return workspace?.metadataVersion;
}
async resetMetadataVersion(workspaceId: string): Promise<void> {
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: 1 },
);
await this.workspaceCacheStorageService.flush(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1);
}
}