admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

4017 improve queries on messages write (#4207)

Browse Source 
* modify code to reduce nested loops and improve performances

* is working

* fix lastSyncHistoryId

* create new service to share it betweent partial sync and full sync

* update partial sync

* update batch limit

* renaming

* adding logs

* update logs

* update logs

* update logs

* delete messages if error while saving the participants

* refactoring

* improving logs

* update logs

* delete historyId if outdated
This commit is contained in:
bosiraphael
2024-02-27 16:06:19 +01:00
committed by GitHub
parent 16fe79b044
commit a19de71fad
9 changed files with 312 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts
View File

@ -15,6 +15,7 @@ import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/w
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
@Injectable()
export class GmailPartialSyncService {
@ -29,6 +30,7 @@ export class GmailPartialSyncService {
private readonly connectedAccountService: ConnectedAccountService,
private readonly messageChannelService: MessageChannelService,
private readonly messageService: MessageService,
private readonly saveMessagesAndCreateContactsService: SaveMessagesAndCreateContactsService,
) {}
public async fetchConnectedAccountThreads(
@ -36,11 +38,6 @@ export class GmailPartialSyncService {
connectedAccountId: string,
maxResults = 500,
): Promise<void> {
const { dataSource: workspaceDataSource, dataSourceMetadata } =
await this.workspaceDataSourceService.connectedToWorkspaceDataSourceAndReturnMetadata(
workspaceId,
);
const connectedAccount = await this.connectedAccountService.getByIdOrFail(
connectedAccountId,
workspaceId,
@ -68,6 +65,11 @@ export class GmailPartialSyncService {
);
if (error && error.code === 404) {
await this.connectedAccountService.deleteHistoryId(
connectedAccountId,
workspaceId,
);
await this.fallbackToFullSync(workspaceId, connectedAccountId);
return;
@ -104,22 +106,23 @@ export class GmailPartialSyncService {
await this.fetchMessagesByBatchesService.fetchAllMessages(
messageQueries,
accessToken,
'gmail full-sync',
workspaceId,
connectedAccountId,
);
if (messagesToSave.length !== 0) {
await this.messageService.saveMessages(
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
messagesToSave,
dataSourceMetadata,
workspaceDataSource,
connectedAccount,
gmailMessageChannelId,
workspaceId,
gmailMessageChannelId,
'gmail partial-sync',
);
}
if (messagesDeleted.length !== 0) {
await this.messageService.deleteMessages(
workspaceDataSource,
messagesDeleted,
gmailMessageChannelId,
workspaceId,