Improve performance twenty orm (#6691)

## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions

View File

@ -21,7 +21,6 @@ import { TokenService } from 'src/engine/core-modules/auth/services/token.servic
import { GoogleAPIsRequest } from 'src/engine/core-modules/auth/types/google-api-request.type';
import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context';
@Controller('auth/google-apis')
@UseFilters(AuthRestApiExceptionFilter)
@ -31,7 +30,6 @@ export class GoogleAPIsAuthController {
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
private readonly onboardingService: OnboardingService,
private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext,
) {}
@Get()
@ -80,13 +78,7 @@ export class GoogleAPIsAuthController {
const handle = emails[0].value;
const googleAPIsServiceInstance =
await this.loadServiceWithWorkspaceContext.load(
this.googleAPIsService,
workspaceId,
);
await googleAPIsServiceInstance.refreshGoogleRefreshToken({
await this.googleAPIsService.refreshGoogleRefreshToken({
handle,
workspaceMemberId: workspaceMemberId,
workspaceId: workspaceId,
@ -97,13 +89,7 @@ export class GoogleAPIsAuthController {
});
if (userId) {
const onboardingServiceInstance =
await this.loadServiceWithWorkspaceContext.load(
this.onboardingService,
workspaceId,
);
await onboardingServiceInstance.setOnboardingConnectAccountPending({
await this.onboardingService.setOnboardingConnectAccountPending({
userId,
workspaceId,
value: false,

View File

@ -8,7 +8,7 @@ import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decora
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import {
CalendarEventListFetchJob,
CalendarEventsImportJobData,
@ -39,7 +39,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
@Injectable()
export class GoogleAPIsService {
constructor(
private readonly twentyORMManager: TwentyORMManager,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@InjectMessageQueue(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@InjectMessageQueue(MessageQueue.calendarQueue)
@ -82,16 +82,19 @@ export class GoogleAPIsService {
const newOrExistingConnectedAccountId = existingAccountId ?? v4();
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const workspaceDataSource = await this.twentyORMManager.getDatasource();
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace(workspaceId);
await workspaceDataSource.transaction(async (manager: EntityManager) => {
if (!existingAccountId) {
@ -146,7 +149,8 @@ export class GoogleAPIsService {
);
const workspaceMemberRepository =
await this.twentyORMManager.getRepository<WorkspaceMemberWorkspaceEntity>(
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);

View File

@ -1,14 +1,14 @@
import { PassportStrategy } from '@nestjs/passport';
import { Injectable } from '@nestjs/common';
import { PassportStrategy } from '@nestjs/passport';
import { Strategy, VerifyCallback } from 'passport-google-oauth20';
import { Request } from 'express';
import { Strategy, VerifyCallback } from 'passport-google-oauth20';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
export type GoogleRequest = Omit<
Request,
'user' | 'workspace' | 'cacheVersion'
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;

View File

@ -12,7 +12,7 @@ import { EnvironmentService } from 'src/engine/integrations/environment/environm
export type MicrosoftRequest = Omit<
Request,
'user' | 'workspace' | 'cacheVersion'
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;

View File

@ -5,7 +5,7 @@ import { MessageChannelVisibility } from 'src/modules/messaging/common/standard-
export type GoogleAPIsRequest = Omit<
Request,
'user' | 'workspace' | 'cacheVersion'
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;