admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

4020 timebox improve performances on messages read (#4140)

Browse Source 
* adding console.time to monitor queries time

* improve query by removing unnecessary JOIN

* improve queries by removing unnecessary JOINs

* improve queries by removing unnecessary JOINs

* remove console.time

* remove logs

* use groupBy from lodash

* modify SELECT

* Revert "use groupBy from lodash"

This reverts commit 852fd3c19367dff79147490b7eaa9fd348d101a1.

* use WorkspaceDataSourceModule
This commit is contained in:
bosiraphael
2024-02-23 10:18:16 +01:00
committed by GitHub
parent 67e27a69ff
commit 35a2178cde
2 changed files with 61 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/twenty-server/src/core/messaging/timeline-messaging.module.ts
View File

@ -2,11 +2,10 @@ import { Module } from '@nestjs/common';
import { TimelineMessagingResolver } from 'src/core/messaging/timeline-messaging.resolver'; import { TimelineMessagingResolver } from 'src/core/messaging/timeline-messaging.resolver';
import { TimelineMessagingService } from 'src/core/messaging/timeline-messaging.service'; import { TimelineMessagingService } from 'src/core/messaging/timeline-messaging.service';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
@Module({ @Module({
imports: [DataSourceModule, TypeORMModule], imports: [WorkspaceDataSourceModule],
exports: [], exports: [],
providers: [TimelineMessagingResolver, TimelineMessagingService], providers: [TimelineMessagingResolver, TimelineMessagingService],
}) })

137
packages/twenty-server/src/core/messaging/timeline-messaging.service.ts
View File

@ -2,8 +2,7 @@ import { Injectable } from '@nestjs/common';
import { TIMELINE_THREADS_DEFAULT_PAGE_SIZE } from 'src/core/messaging/constants/messaging.constants'; import { TIMELINE_THREADS_DEFAULT_PAGE_SIZE } from 'src/core/messaging/constants/messaging.constants';
import { TimelineThreadsWithTotal } from 'src/core/messaging/dtos/timeline-threads-with-total.dto'; import { TimelineThreadsWithTotal } from 'src/core/messaging/dtos/timeline-threads-with-total.dto';
import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { DataSourceService } from 'src/metadata/data-source/data-source.service';
type TimelineThreadParticipant = { type TimelineThreadParticipant = {
personId: string; personId: string;
@ -18,8 +17,7 @@ type TimelineThreadParticipant = {
@Injectable() @Injectable()
export class TimelineMessagingService { export class TimelineMessagingService {
constructor( constructor(
private readonly dataSourceService: DataSourceService, private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly typeORMService: TypeORMService,
) {} ) {}
async getMessagesFromPersonIds( async getMessagesFromPersonIds(
@ -30,13 +28,8 @@ export class TimelineMessagingService {
): Promise<TimelineThreadsWithTotal> { ): Promise<TimelineThreadsWithTotal> {
const offset = (page - 1) * pageSize; const offset = (page - 1) * pageSize;
const dataSourceMetadata = const dataSourceSchema =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( this.workspaceDataSourceService.getSchemaName(workspaceId);
workspaceId,
);
const workspaceDataSource =
await this.typeORMService.connectToDataSource(dataSourceMetadata);
const messageThreads: const messageThreads:
| { | {
@ -46,29 +39,26 @@ export class TimelineMessagingService {
lastMessageBody: string; lastMessageBody: string;
rowNumber: number; rowNumber: number;
}[] }[]
| undefined = await workspaceDataSource?.query( | undefined = await this.workspaceDataSourceService.executeRawQuery(
` `
SELECT * SELECT id,
"lastMessageReceivedAt",
"lastMessageId",
"lastMessageBody"
FROM FROM
(SELECT "messageThread".id, (SELECT message."messageThreadId" AS id,
MAX(message."receivedAt") AS "lastMessageReceivedAt", MAX(message."receivedAt") AS "lastMessageReceivedAt",
message.id AS "lastMessageId", message.id AS "lastMessageId",
message.text AS "lastMessageBody", message.text AS "lastMessageBody",
ROW_NUMBER() OVER (PARTITION BY "messageThread".id ORDER BY MAX(message."receivedAt") DESC) AS "rowNumber" ROW_NUMBER() OVER (PARTITION BY message."messageThreadId" ORDER BY MAX(message."receivedAt") DESC) AS "rowNumber"
FROM FROM
${dataSourceMetadata.schema}."message" message ${dataSourceSchema}."message" message
LEFT JOIN LEFT JOIN
${dataSourceMetadata.schema}."messageThread" "messageThread" ON "messageThread".id = message."messageThreadId" ${dataSourceSchema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id
LEFT JOIN
${dataSourceMetadata.schema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id
LEFT JOIN
${dataSourceMetadata.schema}."person" person ON person.id = "messageParticipant"."personId"
LEFT JOIN
${dataSourceMetadata.schema}."workspaceMember" "workspaceMember" ON "workspaceMember".id = "messageParticipant"."workspaceMemberId"
WHERE WHERE
person.id = ANY($1) "messageParticipant"."personId" = ANY($1)
GROUP BY GROUP BY
"messageThread".id, message."messageThreadId",
message.id message.id
ORDER BY ORDER BY
message."receivedAt" DESC message."receivedAt" DESC
@ -79,6 +69,7 @@ export class TimelineMessagingService {
OFFSET $3 OFFSET $3
`, `,
[personIds, pageSize, offset], [personIds, pageSize, offset],
workspaceId,
); );
if (!messageThreads) { if (!messageThreads) {
@ -97,22 +88,20 @@ export class TimelineMessagingService {
id: string; id: string;
subject: string; subject: string;
}[] }[]
| undefined = await workspaceDataSource?.query( | undefined = await this.workspaceDataSourceService.executeRawQuery(
` `
SELECT * SELECT *
FROM FROM
(SELECT (SELECT
"messageThread".id, message."messageThreadId" AS id,
message.subject, message.subject,
ROW_NUMBER() OVER (PARTITION BY "messageThread".id ORDER BY MAX(message."receivedAt") ASC) AS "rowNumber" ROW_NUMBER() OVER (PARTITION BY message."messageThreadId" ORDER BY MAX(message."receivedAt") ASC) AS "rowNumber"
FROM FROM
${dataSourceMetadata.schema}."message" message ${dataSourceSchema}."message" message
LEFT JOIN
${dataSourceMetadata.schema}."messageThread" "messageThread" ON "messageThread".id = message."messageThreadId"
WHERE WHERE
"messageThread".id = ANY($1) message."messageThreadId" = ANY($1)
GROUP BY GROUP BY
"messageThread".id, message."messageThreadId",
message.id message.id
ORDER BY ORDER BY
message."receivedAt" DESC message."receivedAt" DESC
@ -121,6 +110,7 @@ export class TimelineMessagingService {
"rowNumber" = 1 "rowNumber" = 1
`, `,
[messageThreadIds], [messageThreadIds],
workspaceId,
); );
const numberOfMessagesInThread: const numberOfMessagesInThread:
@ -128,21 +118,20 @@ export class TimelineMessagingService {
id: string; id: string;
numberOfMessagesInThread: number; numberOfMessagesInThread: number;
}[] }[]
| undefined = await workspaceDataSource?.query( | undefined = await this.workspaceDataSourceService.executeRawQuery(
` `
SELECT SELECT
"messageThread".id, message."messageThreadId" AS id,
COUNT(message.id) AS "numberOfMessagesInThread" COUNT(message.id) AS "numberOfMessagesInThread"
FROM FROM
${dataSourceMetadata.schema}."message" message ${dataSourceSchema}."message" message
LEFT JOIN
${dataSourceMetadata.schema}."messageThread" "messageThread" ON "messageThread".id = message."messageThreadId"
WHERE WHERE
"messageThread".id = ANY($1) message."messageThreadId" = ANY($1)
GROUP BY GROUP BY
"messageThread".id message."messageThreadId"
`, `,
[messageThreadIds], [messageThreadIds],
workspaceId,
); );
const messageThreadsByMessageThreadId: { const messageThreadsByMessageThreadId: {
@ -207,9 +196,9 @@ export class TimelineMessagingService {
workspaceMemberAvatarUrl: string; workspaceMemberAvatarUrl: string;
messageDisplayName: string; messageDisplayName: string;
}[] }[]
| undefined = await workspaceDataSource?.query( | undefined = await this.workspaceDataSourceService.executeRawQuery(
` `
SELECT DISTINCT "messageThread".id, SELECT DISTINCT message."messageThreadId" AS id,
message.id AS "messageId", message.id AS "messageId",
message."receivedAt", message."receivedAt",
message.text, message.text,
@ -225,39 +214,36 @@ export class TimelineMessagingService {
"workspaceMember"."avatarUrl" as "workspaceMemberAvatarUrl", "workspaceMember"."avatarUrl" as "workspaceMemberAvatarUrl",
"messageParticipant"."displayName" as "messageDisplayName" "messageParticipant"."displayName" as "messageDisplayName"
FROM FROM
${dataSourceMetadata.schema}."message" message ${dataSourceSchema}."message" message
LEFT JOIN LEFT JOIN
${dataSourceMetadata.schema}."messageThread" "messageThread" ON "messageThread".id = message."messageThreadId" (SELECT * FROM ${dataSourceSchema}."messageParticipant" WHERE "messageParticipant".role = 'from') "messageParticipant" ON "messageParticipant"."messageId" = message.id
LEFT JOIN LEFT JOIN
(SELECT * FROM ${dataSourceMetadata.schema}."messageParticipant" WHERE "messageParticipant".role = 'from') "messageParticipant" ON "messageParticipant"."messageId" = message.id ${dataSourceSchema}."person" person ON person."id" = "messageParticipant"."personId"
LEFT JOIN LEFT JOIN
${dataSourceMetadata.schema}."person" person ON person."id" = "messageParticipant"."personId" ${dataSourceSchema}."workspaceMember" "workspaceMember" ON "workspaceMember".id = "messageParticipant"."workspaceMemberId"
LEFT JOIN
${dataSourceMetadata.schema}."workspaceMember" "workspaceMember" ON "workspaceMember".id = "messageParticipant"."workspaceMemberId"
WHERE WHERE
"messageThread".id = ANY($1) message."messageThreadId" = ANY($1)
ORDER BY ORDER BY
message."receivedAt" DESC message."receivedAt" DESC
`, `,
[messageThreadIds], [messageThreadIds],
workspaceId,
); );
const totalNumberOfThreads = await workspaceDataSource?.query( const totalNumberOfThreads =
` await this.workspaceDataSourceService.executeRawQuery(
SELECT COUNT(DISTINCT "messageThread".id) `
SELECT COUNT(DISTINCT message."messageThreadId")
FROM FROM
${dataSourceMetadata.schema}."message" message ${dataSourceSchema}."message" message
LEFT JOIN LEFT JOIN
${dataSourceMetadata.schema}."messageThread" "messageThread" ON "messageThread".id = message."messageThreadId" ${dataSourceSchema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id
LEFT JOIN
${dataSourceMetadata.schema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id
LEFT JOIN
${dataSourceMetadata.schema}."person" person ON person.id = "messageParticipant"."personId"
WHERE WHERE
person.id = ANY($1) "messageParticipant"."personId" = ANY($1)
`, `,
[personIds], [personIds],
); workspaceId,
);
const threadParticipantsByThreadId: { const threadParticipantsByThreadId: {
[key: string]: TimelineThreadParticipant[]; [key: string]: TimelineThreadParticipant[];
@ -322,23 +308,22 @@ export class TimelineMessagingService {
id: string; id: string;
visibility: 'metadata' | 'subject' | 'share_everything'; visibility: 'metadata' | 'subject' | 'share_everything';
}[] }[]
| undefined = await workspaceDataSource?.query( | undefined = await this.workspaceDataSourceService.executeRawQuery(
` `
SELECT SELECT
"messageThread".id, message."messageThreadId" AS id,
"messageChannel".visibility "messageChannel".visibility
FROM FROM
${dataSourceMetadata.schema}."messageThread" "messageThread" ${dataSourceSchema}."message" message
LEFT JOIN LEFT JOIN
${dataSourceMetadata.schema}."message" message ON "message"."messageThreadId" = "messageThread".id ${dataSourceSchema}."messageChannelMessageAssociation" "messageChannelMessageAssociation" ON "messageChannelMessageAssociation"."messageId" = message.id
LEFT JOIN LEFT JOIN
${dataSourceMetadata.schema}."messageChannelMessageAssociation" "messageChannelMessageAssociation" ON "messageChannelMessageAssociation"."messageId" = message.id ${dataSourceSchema}."messageChannel" "messageChannel" ON "messageChannel".id = "messageChannelMessageAssociation"."messageChannelId"
LEFT JOIN
${dataSourceMetadata.schema}."messageChannel" "messageChannel" ON "messageChannel".id = "messageChannelMessageAssociation"."messageChannelId"
WHERE WHERE
"messageThread".id = ANY($1) message."messageThreadId" = ANY($1)
`, `,
[messageThreadIds], [messageThreadIds],
workspaceId,
); );
const visibilityValues = ['metadata', 'subject', 'share_everything']; const visibilityValues = ['metadata', 'subject', 'share_everything'];
@ -430,24 +415,20 @@ export class TimelineMessagingService {
page: number = 1, page: number = 1,
pageSize: number = TIMELINE_THREADS_DEFAULT_PAGE_SIZE, pageSize: number = TIMELINE_THREADS_DEFAULT_PAGE_SIZE,
): Promise<TimelineThreadsWithTotal> { ): Promise<TimelineThreadsWithTotal> {
const dataSourceMetadata = const dataSourceSchema =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( this.workspaceDataSourceService.getSchemaName(workspaceId);
workspaceId,
);
const workspaceDataSource = const personIds = await this.workspaceDataSourceService.executeRawQuery(
await this.typeORMService.connectToDataSource(dataSourceMetadata);
const personIds = await workspaceDataSource?.query(
` `
SELECT SELECT
p."id" p."id"
FROM FROM
${dataSourceMetadata.schema}."person" p ${dataSourceSchema}."person" p
WHERE WHERE
p."companyId" = $1 p."companyId" = $1
`, `,
[companyId], [companyId],
workspaceId,
); );
if (!personIds) { if (!personIds) {