admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance twenty orm (#6691)

Browse Source 
## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts
View File

@ -21,7 +21,6 @@ import { TokenService } from 'src/engine/core-modules/auth/services/token.servic
import { GoogleAPIsRequest } from 'src/engine/core-modules/auth/types/google-api-request.type';
import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context';
@Controller('auth/google-apis')
@UseFilters(AuthRestApiExceptionFilter)
@ -31,7 +30,6 @@ export class GoogleAPIsAuthController {
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
private readonly onboardingService: OnboardingService,
private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext,
) {}
@Get()
@ -80,13 +78,7 @@ export class GoogleAPIsAuthController {
const handle = emails[0].value;
const googleAPIsServiceInstance =
await this.loadServiceWithWorkspaceContext.load(
this.googleAPIsService,
workspaceId,
);
await googleAPIsServiceInstance.refreshGoogleRefreshToken({
await this.googleAPIsService.refreshGoogleRefreshToken({
handle,
workspaceMemberId: workspaceMemberId,
workspaceId: workspaceId,
@ -97,13 +89,7 @@ export class GoogleAPIsAuthController {
});
if (userId) {
const onboardingServiceInstance =
await this.loadServiceWithWorkspaceContext.load(
this.onboardingService,
workspaceId,
);
await onboardingServiceInstance.setOnboardingConnectAccountPending({
await this.onboardingService.setOnboardingConnectAccountPending({
userId,
workspaceId,
value: false,

16
packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts
View File

@ -8,7 +8,7 @@ import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decora
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import {
CalendarEventListFetchJob,
CalendarEventsImportJobData,
@ -39,7 +39,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
@Injectable()
export class GoogleAPIsService {
constructor(
private readonly twentyORMManager: TwentyORMManager,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@InjectMessageQueue(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@InjectMessageQueue(MessageQueue.calendarQueue)
@ -82,16 +82,19 @@ export class GoogleAPIsService {
const newOrExistingConnectedAccountId = existingAccountId ?? v4();
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const workspaceDataSource = await this.twentyORMManager.getDatasource();
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace(workspaceId);
await workspaceDataSource.transaction(async (manager: EntityManager) => {
if (!existingAccountId) {
@ -146,7 +149,8 @@ export class GoogleAPIsService {
);
const workspaceMemberRepository =
await this.twentyORMManager.getRepository<WorkspaceMemberWorkspaceEntity>(
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);

6
packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts
View File

@ -1,14 +1,14 @@
import { PassportStrategy } from '@nestjs/passport';
import { Injectable } from '@nestjs/common';
import { PassportStrategy } from '@nestjs/passport';
import { Strategy, VerifyCallback } from 'passport-google-oauth20';
import { Request } from 'express';
import { Strategy, VerifyCallback } from 'passport-google-oauth20';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
export type GoogleRequest = Omit<
Request,
'user' | 'workspace' | 'cacheVersion'
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;

2
packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts
View File

@ -12,7 +12,7 @@ import { EnvironmentService } from 'src/engine/integrations/environment/environm
export type MicrosoftRequest = Omit<
Request,
'user' | 'workspace' | 'cacheVersion'
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;

2
packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts
View File

@ -5,7 +5,7 @@ import { MessageChannelVisibility } from 'src/modules/messaging/common/standard-
export type GoogleAPIsRequest = Omit<
Request,
'user' | 'workspace' | 'cacheVersion'
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;

4
packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts
View File

@ -44,6 +44,10 @@ export class FileController {
workspaceId,
);
fileStream.on('error', () => {
res.status(500).send({ error: 'Internal server error' });
});
fileStream.pipe(res);
} catch (error) {
if (

8
packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts
View File

@ -119,13 +119,13 @@ export const useGraphQLErrorHandlerHook = <
if (Array.isArray(errors) && errors.length > 0) {
const headers = context.req.headers;
const currentSchemaVersion = context.req.cacheVersion;
const currentMetadataVersion = context.req.workspaceMetadataVersion;
const requestSchemaVersion = headers['x-schema-version'];
const requestMetadataVersion = headers['x-schema-version'];
if (
requestSchemaVersion &&
requestSchemaVersion !== currentSchemaVersion
requestMetadataVersion &&
requestMetadataVersion !== `${currentMetadataVersion}`
) {
throw new GraphQLError(
`Schema version mismatch, please refresh the page.`,

9
packages/twenty-server/src/engine/core-modules/user/user.resolver.ts
View File

@ -33,7 +33,6 @@ import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorat
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
const getHMACKey = (email?: string, key?: string | null) => {
@ -54,7 +53,6 @@ export class UserResolver {
private readonly environmentService: EnvironmentService,
private readonly fileUploadService: FileUploadService,
private readonly onboardingService: OnboardingService,
private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext,
private readonly userVarService: UserVarsService,
private readonly fileService: FileService,
) {}
@ -189,11 +187,6 @@ export class UserResolver {
@ResolveField(() => OnboardingStatus)
async onboardingStatus(@Parent() user: User): Promise<OnboardingStatus> {
const contextInstance = await this.loadServiceWithWorkspaceContext.load(
this.onboardingService,
user.defaultWorkspaceId,
);
return contextInstance.getOnboardingStatus(user);
return this.onboardingService.getOnboardingStatus(user);
}
}

12
packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts
View File

@ -117,4 +117,16 @@ export class Workspace {
(postgresCredentials) => postgresCredentials.workspace,
)
allPostgresCredentials: Relation<PostgresCredentials[]>;
@Field()
@Column({ default: 1 })
metadataVersion: number;
@Field()
@Column({ default: '' })
databaseUrl: string;
@Field()
@Column({ default: '' })
databaseSchema: string;
}

4
packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts
View File

@ -16,7 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity';
import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener';
import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts';
@ -32,7 +32,7 @@ import { WorkspaceService } from './services/workspace.service';
BillingModule,
FileModule,
FileUploadModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
NestjsQueryTypeOrmModule.forFeature(
[User, Workspace, UserWorkspace, FeatureFlagEntity],
'core',

23
packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts
View File

@ -26,7 +26,7 @@ import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { assert } from 'src/utils/assert';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
@ -39,7 +39,7 @@ import { WorkspaceService } from './services/workspace.service';
export class WorkspaceResolver {
constructor(
private readonly workspaceService: WorkspaceService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly userWorkspaceService: UserWorkspaceService,
private readonly fileUploadService: FileUploadService,
private readonly fileService: FileService,
@ -107,13 +107,6 @@ export class WorkspaceResolver {
return this.workspaceService.deleteWorkspace(id);
}
@ResolveField(() => String, { nullable: true })
async currentCacheVersion(
@Parent() workspace: Workspace,
): Promise<string | null> {
return this.workspaceCacheVersionService.getVersion(workspace.id);
}
@ResolveField(() => BillingSubscription, { nullable: true })
async currentBillingSubscription(
@Parent() workspace: Workspace,
@ -133,11 +126,15 @@ export class WorkspaceResolver {
@ResolveField(() => String)
async logo(@Parent() workspace: Workspace): Promise<string> {
if (workspace.logo) {
const workspaceLogoToken = await this.fileService.encodeFileToken({
workspace_id: workspace.id,
});
try {
const workspaceLogoToken = await this.fileService.encodeFileToken({
workspace_id: workspace.id,
});
return `${workspace.logo}?token=${workspaceLogoToken}`;
return `${workspace.logo}?token=${workspaceLogoToken}`;
} catch (e) {
return workspace.logo;
}
}
return workspace.logo ?? '';