admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix race condition with datasource creation (#7106)

Browse Source 
## Context
We currently have a race condition when dealing with datasource
creation. This happen when multiple queries arrive at the same time (for
example graphql dataloaders) and the datasource is not created yet.
Since the datasource is stored in memory this can happen more often as
well and they were all triggering the datasource creation at the same
time.

I'm trying to fix the issue with promise memoization. Now, instead of
caching the datasource only, we also want to cache the promise of the
datasource creation and make the creation itself synchronous.

More info about promise memoization in this article for example:
https://www.jonmellman.com/posts/promise-memoization

Co-authored-by: Charles Bochet <charlesBochet@users.noreply.github.com>
This commit is contained in:
Weiko
2024-09-18 14:01:55 +02:00
committed by GitHub
parent 94ba4c0558
commit 999974893c
1 changed files with 134 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -18,6 +18,7 @@ import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage
export class WorkspaceDatasourceFactory { export class WorkspaceDatasourceFactory {
private readonly logger = new Logger(WorkspaceDatasourceFactory.name); private readonly logger = new Logger(WorkspaceDatasourceFactory.name);
private cacheManager = new CacheManager<WorkspaceDataSource>(); private cacheManager = new CacheManager<WorkspaceDataSource>();
private cachedDatasourcePromise: Record<string, Promise<WorkspaceDataSource>>;
constructor( constructor(
private readonly dataSourceService: DataSourceService, private readonly dataSourceService: DataSourceService,
@ -25,7 +26,9 @@ export class WorkspaceDatasourceFactory {
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
private readonly entitySchemaFactory: EntitySchemaFactory, private readonly entitySchemaFactory: EntitySchemaFactory,
) {} ) {
this.cachedDatasourcePromise = {};
}
public async create( public async create(
workspaceId: string, workspaceId: string,
@ -37,8 +40,16 @@ export class WorkspaceDatasourceFactory {
workspaceMetadataVersion, workspaceMetadataVersion,
); );
const workspaceDataSource = await this.cacheManager.execute( const cacheKey = `${workspaceId}-${desiredWorkspaceMetadataVersion}`;
`${workspaceId}-${desiredWorkspaceMetadataVersion}`,
if (cacheKey in this.cachedDatasourcePromise) {
return this.cachedDatasourcePromise[cacheKey];
}
const creationPromise = (async (): Promise<WorkspaceDataSource> => {
try {
const result = await this.cacheManager.execute(
cacheKey as '`${string}-${string}`',
async () => { async () => {
this.logger.log( this.logger.log(
`Creating workspace data source for workspace ${workspaceId} and metadata version ${desiredWorkspaceMetadataVersion}`, `Creating workspace data source for workspace ${workspaceId} and metadata version ${desiredWorkspaceMetadataVersion}`,
@ -147,11 +158,21 @@ export class WorkspaceDatasourceFactory {
}, },
); );
if (!workspaceDataSource) { if (result === null) {
throw new Error('Workspace data source not found'); throw new Error(
`Failed to create WorkspaceDataSource for ${cacheKey}`,
);
} }
return workspaceDataSource; return result;
} finally {
delete this.cachedDatasourcePromise[cacheKey];
}
})();
this.cachedDatasourcePromise[cacheKey] = creationPromise;
return creationPromise;
} }
private async computeDesiredWorkspaceMetadataVersion( private async computeDesiredWorkspaceMetadataVersion(