admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance twenty orm (#6691)

Browse Source 
## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts
View File

@ -32,6 +32,8 @@ export class CalendarEventListFetchCronJob {
@Process(CalendarEventListFetchCronJob.name)
async handle(): Promise<void> {
console.time('CalendarEventListFetchCronJob time');
const activeWorkspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
@ -65,5 +67,7 @@ export class CalendarEventListFetchCronJob {
);
}
}
console.timeEnd('CalendarEventListFetchCronJob time');
}
}

3
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts
View File

@ -33,6 +33,8 @@ export class CalendarEventListFetchJob {
@Process(CalendarEventListFetchJob.name)
async handle(data: CalendarEventsImportJobData): Promise<void> {
console.time('CalendarEventListFetchJob time');
const { workspaceId, calendarChannelId } = data;
const calendarChannelRepository =
@ -91,5 +93,6 @@ export class CalendarEventListFetchJob {
default:
break;
}
console.timeEnd('CalendarEventListFetchJob time');
}
}

2
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts
View File

@ -16,7 +16,7 @@ import { AccountsToReconnectKeys } from 'src/modules/connected-account/types/acc
export class CalendarChannelSyncStatusService {
constructor(
private readonly twentyORMManager: TwentyORMManager,
@InjectCacheStorage(CacheStorageNamespace.Calendar)
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly accountsToReconnectService: AccountsToReconnectService,
) {}

2
packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts
View File

@ -15,7 +15,7 @@ import {
@Injectable()
export class MessagingChannelSyncStatusService {
constructor(
@InjectCacheStorage(CacheStorageNamespace.Messaging)
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
private readonly twentyORMManager: TwentyORMManager,
private readonly accountsToReconnectService: AccountsToReconnectService,

4
packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts
View File

@ -33,6 +33,8 @@ export class MessagingMessageListFetchCronJob {
@Process(MessagingMessageListFetchCronJob.name)
async handle(): Promise<void> {
console.time('MessagingMessageListFetchCronJob time');
const activeWorkspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
@ -66,5 +68,7 @@ export class MessagingMessageListFetchCronJob {
}
}
}
console.timeEnd('MessagingMessageListFetchCronJob time');
}
}

3
packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts
View File

@ -36,6 +36,7 @@ export class MessagingMessagesImportCronJob {
@Process(MessagingMessagesImportCronJob.name)
async handle(): Promise<void> {
console.time('MessagingMessagesImportCronJob time');
const activeWorkspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
@ -69,5 +70,7 @@ export class MessagingMessagesImportCronJob {
}
}
}
console.timeEnd('MessagingMessagesImportCronJob time');
}
}

2
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts
View File

@ -14,7 +14,7 @@ export type MessagingAddSingleMessageToCacheForImportJobData = {
@Processor(MessageQueue.messagingQueue)
export class MessagingAddSingleMessageToCacheForImportJob {
constructor(
@InjectCacheStorage(CacheStorageNamespace.Messaging)
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
) {}

10
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts
View File

@ -1,11 +1,11 @@
import { Logger } from '@nestjs/common';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
export type MessagingCleanCacheJobData = {
workspaceId: string;
@ -17,7 +17,7 @@ export class MessagingCleanCacheJob {
private readonly logger = new Logger(MessagingCleanCacheJob.name);
constructor(
@InjectCacheStorage(CacheStorageNamespace.Messaging)
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
) {}

4
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts
View File

@ -39,6 +39,8 @@ export class MessagingMessageListFetchJob {
@Process(MessagingMessageListFetchJob.name)
async handle(data: MessagingMessageListFetchJobData): Promise<void> {
console.time('MessagingMessageListFetchJob time');
const { messageChannelId, workspaceId } = data;
await this.messagingTelemetryService.track({
@ -145,5 +147,7 @@ export class MessagingMessageListFetchJob {
default:
break;
}
console.timeEnd('MessagingMessageListFetchJob time');
}
}

4
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts
View File

@ -35,6 +35,8 @@ export class MessagingMessagesImportJob {
@Process(MessagingMessagesImportJob.name)
async handle(data: MessagingMessagesImportJobData): Promise<void> {
console.time('MessagingMessagesImportJob time');
const { messageChannelId, workspaceId } = data;
await this.messagingTelemetryService.track({
@ -95,5 +97,7 @@ export class MessagingMessagesImportJob {
connectedAccount,
workspaceId,
);
console.timeEnd('MessagingMessagesImportJob time');
}
}

2
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts
View File

@ -29,7 +29,7 @@ export class MessagingFullMessageListFetchService {
constructor(
private readonly gmailClientProvider: MessagingGmailClientProvider,
@InjectCacheStorage(CacheStorageNamespace.Messaging)
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService,
private readonly gmailErrorHandlingService: MessagingErrorHandlingService,

2
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
View File

@ -31,7 +31,7 @@ export class MessagingMessagesImportService {
constructor(
private readonly fetchMessagesByBatchesService: MessagingGmailFetchMessagesByBatchesService,
@InjectCacheStorage(CacheStorageNamespace.Messaging)
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService,
private readonly saveMessagesAndEnqueueContactCreationService: MessagingSaveMessagesAndEnqueueContactCreationService,

2
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts
View File

@ -24,7 +24,7 @@ export class MessagingPartialMessageListFetchService {
constructor(
private readonly gmailClientProvider: MessagingGmailClientProvider,
@InjectCacheStorage(CacheStorageNamespace.Messaging)
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
private readonly gmailErrorHandlingService: MessagingErrorHandlingService,
private readonly gmailGetHistoryService: MessagingGmailHistoryService,

4
packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts
View File

@ -32,6 +32,8 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
async handle(): Promise<void> {
this.logger.log('Starting message channel sync status monitoring...');
console.time('MessagingMessageChannelSyncStatusMonitoringCronJob time');
await this.messagingTelemetryService.track({
eventName: 'message_channel.monitoring.sync_status.start',
message: 'Starting message channel sync status monitoring',
@ -68,5 +70,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
});
}
}
console.timeEnd('MessagingMessageChannelSyncStatusMonitoringCronJob time');
}
}