admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance twenty orm (#6691)

Browse Source 
## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts
View File

@ -1,11 +1,12 @@
import { Test, TestingModule } from '@nestjs/testing';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory';
import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
describe('WorkspaceSchemaFactory', () => {
@ -39,6 +40,10 @@ describe('WorkspaceSchemaFactory', () => {
provide: WorkspaceCacheStorageService,
useValue: {},
},
{
provide: WorkspaceMetadataVersionService,
useValue: {},
},
],
}).compile();

6
packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts
View File

@ -1,10 +1,11 @@
import { Module } from '@nestjs/common';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module';
import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module';
import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module';
import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { WorkspaceSchemaFactory } from './workspace-schema.factory';
@ -21,6 +22,7 @@ import { WorkspaceSchemaFactory } from './workspace-schema.factory';
WorkspaceSchemaBuilderModule,
WorkspaceResolverBuilderModule,
WorkspaceCacheStorageModule,
WorkspaceMetadataVersionModule,
],
providers: [WorkspaceSchemaFactory, ScalarsExplorerService],
exports: [WorkspaceSchemaFactory],

5
packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts
View File

@ -9,10 +9,11 @@ export type CacheMetadataPluginConfig = {
export function useCachedMetadata(config: CacheMetadataPluginConfig): Plugin {
const computeCacheKey = (serverContext: any) => {
const workspaceId = serverContext.req.workspace?.id ?? 'anonymous';
const cacheVersion = serverContext.req.cacheVersion ?? '0';
const workspaceMetadataVersion =
serverContext.req.workspaceMetadataVersion ?? '0';
const operationName = getOperationName(serverContext);
return `cachedOperations:${operationName}:${workspaceId}:${cacheVersion}`;
return `graphql:operations:${operationName}:${workspaceId}:${workspaceMetadataVersion}`;
};
const getOperationName = (serverContext: any) =>

16
packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts
View File

@ -1,19 +1,18 @@
import { Module } from '@nestjs/common';
import { GraphQLModule } from '@nestjs/graphql';
import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs';
import { YogaDriver, YogaDriverConfig } from '@graphql-yoga/nestjs';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module';
import { metadataModuleFactory } from 'src/engine/api/graphql/metadata.module-factory';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service';
import { DataloaderModule } from 'src/engine/dataloaders/dataloader.module';
import { DataloaderService } from 'src/engine/dataloaders/dataloader.service';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
import { CacheStorageModule } from 'src/engine/integrations/cache-storage/cache-storage.module';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service';
import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
@Module({
imports: [
@ -25,13 +24,12 @@ import { CacheStorageModule } from 'src/engine/integrations/cache-storage/cache-
EnvironmentService,
ExceptionHandlerService,
DataloaderService,
CacheStorageNamespace.WorkspaceSchema,
CacheStorageNamespace.EngineWorkspace,
],
}),
MetadataEngineModule,
WorkspaceMigrationRunnerModule,
WorkspaceMigrationModule,
CacheStorageModule,
],
})
export class MetadataGraphQLApiModule {}

10
packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts
View File

@ -15,7 +15,7 @@ export const metadataModuleFactory = async (
environmentService: EnvironmentService,
exceptionHandlerService: ExceptionHandlerService,
dataloaderService: DataloaderService,
workspaceSchemaCacheStorage: CacheStorageService,
cacheStorageService: CacheStorageService,
): Promise<YogaDriverConfig> => {
const config: YogaDriverConfig = {
autoSchemaFile: true,
@ -36,12 +36,8 @@ export const metadataModuleFactory = async (
exceptionHandlerService,
}),
useCachedMetadata({
cacheGetter: workspaceSchemaCacheStorage.get.bind(
workspaceSchemaCacheStorage,
),
cacheSetter: workspaceSchemaCacheStorage.set.bind(
workspaceSchemaCacheStorage,
),
cacheGetter: cacheStorageService.get.bind(cacheStorageService),
cacheSetter: cacheStorageService.set.bind(cacheStorageService),
operationsToCache: ['ObjectMetadataItems'],
}),
],

22
packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts
View File

@ -1,17 +1,18 @@
import { Injectable } from '@nestjs/common';
import { GraphQLSchema, printSchema } from 'graphql';
import { makeExecutableSchema } from '@graphql-tools/schema';
import { GraphQLSchema, printSchema } from 'graphql';
import { gql } from 'graphql-tag';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories';
import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory';
import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceSchemaFactory {
@ -22,6 +23,7 @@ export class WorkspaceSchemaFactory {
private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory,
private readonly workspaceResolverFactory: WorkspaceResolverFactory,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
) {}
async createGraphQLSchema(authContext: AuthContext): Promise<GraphQLSchema> {
@ -40,7 +42,7 @@ export class WorkspaceSchemaFactory {
}
// Validate cache version
await this.workspaceCacheStorageService.validateCacheVersion(
await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated(
authContext.workspace.id,
);
@ -64,11 +66,11 @@ export class WorkspaceSchemaFactory {
}
// Get typeDefs from cache
let typeDefs = await this.workspaceCacheStorageService.getTypeDefs(
let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs(
authContext.workspace.id,
);
let usedScalarNames =
await this.workspaceCacheStorageService.getUsedScalarNames(
await this.workspaceCacheStorageService.getGraphQLUsedScalarNames(
authContext.workspace.id,
);
@ -84,11 +86,11 @@ export class WorkspaceSchemaFactory {
this.scalarsExplorerService.getUsedScalarNames(autoGeneratedSchema);
typeDefs = printSchema(autoGeneratedSchema);
await this.workspaceCacheStorageService.setTypeDefs(
await this.workspaceCacheStorageService.setGraphQLTypeDefs(
authContext.workspace.id,
typeDefs,
);
await this.workspaceCacheStorageService.setUsedScalarNames(
await this.workspaceCacheStorageService.setGraphQLUsedScalarNames(
authContext.workspace.id,
usedScalarNames,
);