admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Messaging] Gmail Full sync pagination (#3664)

Browse Source
This commit is contained in:
Weiko
2024-01-29 11:57:54 +01:00
committed by GitHub
parent a654205dbc
commit d66d8c9907
11 changed files with 128 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts
View File

@ -6,12 +6,10 @@ import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { GmailFullSyncCommand } from 'src/workspace/messaging/commands/gmail-full-sync.command';
import { GmailPartialSyncCommand } from 'src/workspace/messaging/commands/gmail-partial-sync.command';
import { MessagingModule } from 'src/workspace/messaging/messaging.module';
import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
@Module({
imports: [
MessagingModule,
DataSourceModule,
TypeORMModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),

34
packages/twenty-server/src/workspace/messaging/commands/gmail-full-sync.command.ts
View File

@ -1,4 +1,5 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Inject } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { Repository } from 'typeorm';
@ -7,8 +8,13 @@ import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/core/feature-flag/feature-flag.entity';
import { MessagingProducer } from 'src/workspace/messaging/producers/messaging-producer';
import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import {
GmailFullSyncJobData,
GmailFullSyncJob,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
interface GmailFullSyncOptions {
workspaceId: string;
@ -20,11 +26,11 @@ interface GmailFullSyncOptions {
})
export class GmailFullSyncCommand extends CommandRunner {
constructor(
private readonly messagingProducer: MessagingProducer,
private readonly utils: MessagingUtilsService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();
}
@ -58,13 +64,25 @@ export class GmailFullSyncCommand extends CommandRunner {
}
private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
const connectedAccounts =
await this.utils.getConnectedAccountsFromWorkspaceId(workspaceId);
const { workspaceDataSource, dataSourceMetadata } =
await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
const connectedAccounts = await this.utils.getConnectedAccounts(
dataSourceMetadata,
workspaceDataSource,
);
for (const connectedAccount of connectedAccounts) {
await this.messagingProducer.enqueueGmailFullSync(
{ workspaceId, connectedAccountId: connectedAccount.id },
`${workspaceId}-${connectedAccount.id}`,
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
{
workspaceId,
connectedAccountId: connectedAccount.id,
},
{
id: `${workspaceId}-${connectedAccount.id}`,
retryLimit: 2,
},
);
}
}

34
packages/twenty-server/src/workspace/messaging/commands/gmail-partial-sync.command.ts
View File

@ -1,4 +1,5 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Inject } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { Repository } from 'typeorm';
@ -7,8 +8,13 @@ import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/core/feature-flag/feature-flag.entity';
import { MessagingProducer } from 'src/workspace/messaging/producers/messaging-producer';
import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import {
GmailPartialSyncJob,
GmailPartialSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-partial-sync.job';
interface GmailPartialSyncOptions {
workspaceId: string;
@ -20,11 +26,11 @@ interface GmailPartialSyncOptions {
})
export class GmailPartialSyncCommand extends CommandRunner {
constructor(
private readonly messagingProducer: MessagingProducer,
private readonly utils: MessagingUtilsService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();
}
@ -58,13 +64,25 @@ export class GmailPartialSyncCommand extends CommandRunner {
}
private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
const connectedAccounts =
await this.utils.getConnectedAccountsFromWorkspaceId(workspaceId);
const { workspaceDataSource, dataSourceMetadata } =
await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
const connectedAccounts = await this.utils.getConnectedAccounts(
dataSourceMetadata,
workspaceDataSource,
);
for (const connectedAccount of connectedAccounts) {
await this.messagingProducer.enqueueGmailPartialSync(
{ workspaceId, connectedAccountId: connectedAccount.id },
`${workspaceId}-${connectedAccount.id}`,
await this.messageQueueService.add<GmailPartialSyncJobData>(
GmailPartialSyncJob.name,
{
workspaceId,
connectedAccountId: connectedAccount.id,
},
{
id: `${workspaceId}-${connectedAccount.id}`,
retryLimit: 2,
},
);
}
}

6
packages/twenty-server/src/workspace/messaging/jobs/gmail-full-sync.job.ts
View File

@ -9,6 +9,7 @@ import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-ful
export type GmailFullSyncJobData = {
workspaceId: string;
connectedAccountId: string;
nextPageToken?: string;
};
@Injectable()
@ -21,8 +22,10 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
async handle(data: GmailFullSyncJobData): Promise<void> {
console.log(
`fetching messages for workspace ${data.workspaceId} and account ${
`gmail full-sync for workspace ${data.workspaceId} and account ${
data.connectedAccountId
} ${
data.nextPageToken ? `and ${data.nextPageToken} pageToken` : ''
} with ${this.environmentService.getMessageQueueDriverType()}`,
);
await this.gmailRefreshAccessTokenService.refreshAndSaveAccessToken(
@ -33,6 +36,7 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
await this.fetchWorkspaceMessagesService.fetchConnectedAccountThreads(
data.workspaceId,
data.connectedAccountId,
data.nextPageToken,
);
}
}

2
packages/twenty-server/src/workspace/messaging/jobs/gmail-partial-sync.job.ts
View File

@ -23,7 +23,7 @@ export class GmailPartialSyncJob
async handle(data: GmailPartialSyncJobData): Promise<void> {
console.log(
`fetching messages for workspace ${data.workspaceId} and account ${
`gmail partial-sync for workspace ${data.workspaceId} and account ${
data.connectedAccountId
} with ${this.environmentService.getMessageQueueDriverType()}`,
);

10
packages/twenty-server/src/workspace/messaging/messaging.module.ts
View File

@ -1,10 +0,0 @@
import { Module } from '@nestjs/common';
import { MessagingProducer } from 'src/workspace/messaging/producers/messaging-producer';
@Module({
imports: [],
providers: [MessagingProducer],
exports: [MessagingProducer],
})
export class MessagingModule {}

45
packages/twenty-server/src/workspace/messaging/producers/messaging-producer.ts
View File

@ -1,45 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import {
GmailFullSyncJob,
GmailFullSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import {
GmailPartialSyncJob,
GmailPartialSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-partial-sync.job';
@Injectable()
export class MessagingProducer {
constructor(
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
) {}
async enqueueGmailFullSync(data: GmailFullSyncJobData, singletonKey: string) {
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
data,
{
id: singletonKey,
retryLimit: 2,
},
);
}
async enqueueGmailPartialSync(
data: GmailPartialSyncJobData,
singletonKey: string,
) {
await this.messageQueueService.add<GmailPartialSyncJobData>(
GmailPartialSyncJob.name,
data,
{
id: singletonKey,
retryLimit: 2,
},
);
}
}

8
packages/twenty-server/src/workspace/messaging/services/fetch-workspace-messages.module.ts
View File

@ -3,7 +3,6 @@ import { Module } from '@nestjs/common';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { EnvironmentModule } from 'src/integrations/environment/environment.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { MessagingModule } from 'src/workspace/messaging/messaging.module';
import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service';
import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service';
@ -12,12 +11,7 @@ import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services
import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
@Module({
imports: [
MessagingModule,
TypeORMModule,
DataSourceModule,
EnvironmentModule,
],
imports: [TypeORMModule, DataSourceModule, EnvironmentModule],
providers: [
GmailFullSyncService,
GmailPartialSyncService,

43
packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts
View File

@ -1,8 +1,14 @@
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service';
import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import {
GmailFullSyncJobData,
GmailFullSyncJob,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
@Injectable()
export class GmailFullSyncService {
@ -10,18 +16,23 @@ export class GmailFullSyncService {
private readonly gmailClientProvider: GmailClientProvider,
private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
private readonly utils: MessagingUtilsService,
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
) {}
public async fetchConnectedAccountThreads(
workspaceId: string,
connectedAccountId: string,
maxResults = 500,
nextPageToken?: string,
): Promise<void> {
const { workspaceDataSource, dataSourceMetadata, connectedAccount } =
await this.utils.getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
workspaceId,
connectedAccountId,
);
const { workspaceDataSource, dataSourceMetadata } =
await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
const connectedAccount = await this.utils.getConnectedAcountByIdOrFail(
connectedAccountId,
dataSourceMetadata,
workspaceDataSource,
);
const accessToken = connectedAccount.accessToken;
const refreshToken = connectedAccount.refreshToken;
@ -48,7 +59,8 @@ export class GmailFullSyncService {
const messages = await gmailClient.users.messages.list({
userId: 'me',
maxResults,
maxResults: 500,
pageToken: nextPageToken,
});
const messagesData = messages.data.messages;
@ -119,5 +131,20 @@ export class GmailFullSyncService {
dataSourceMetadata,
workspaceDataSource,
);
if (messages.data.nextPageToken) {
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
{
workspaceId,
connectedAccountId,
nextPageToken: messages.data.nextPageToken,
},
{
id: `${workspaceId}-${connectedAccountId}`,
retryLimit: 2,
},
);
}
}
}

27
packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts
View File

@ -28,11 +28,14 @@ export class GmailPartialSyncService {
lastSyncHistoryId: string,
maxResults: number,
) {
const { connectedAccount } =
await this.utils.getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
workspaceId,
connectedAccountId,
);
const { workspaceDataSource, dataSourceMetadata } =
await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
const connectedAccount = await this.utils.getConnectedAcountByIdOrFail(
connectedAccountId,
dataSourceMetadata,
workspaceDataSource,
);
const gmailClient = await this.gmailClientProvider.getGmailClient(
connectedAccount.refreshToken,
@ -53,17 +56,19 @@ export class GmailPartialSyncService {
connectedAccountId: string,
maxResults = 500,
): Promise<void> {
const { workspaceDataSource, dataSourceMetadata, connectedAccount } =
await this.utils.getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
workspaceId,
connectedAccountId,
);
const { workspaceDataSource, dataSourceMetadata } =
await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
const connectedAccount = await this.utils.getConnectedAcountByIdOrFail(
connectedAccountId,
dataSourceMetadata,
workspaceDataSource,
);
const lastSyncHistoryId = connectedAccount.lastSyncHistoryId;
if (!lastSyncHistoryId) {
// Fall back to full sync
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
{ workspaceId, connectedAccountId },

46
packages/twenty-server/src/workspace/messaging/services/messaging-utils.service.ts
View File

@ -202,39 +202,39 @@ export class MessagingUtilsService {
);
}
public async getConnectedAccountsFromWorkspaceId(
workspaceId: string,
public async getConnectedAccounts(
dataSourceMetadata: DataSourceEntity,
workspaceDataSource: DataSource,
): Promise<any[]> {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
workspaceId,
);
const workspaceDataSource =
await this.typeORMService.connectToDataSource(dataSourceMetadata);
if (!workspaceDataSource) {
throw new Error('No workspace data source found');
}
const connectedAccounts = await workspaceDataSource?.query(
`SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'google'`,
);
return connectedAccounts;
}
public async getConnectedAcountByIdOrFail(
connectedAccountId: string,
dataSourceMetadata: DataSourceEntity,
workspaceDataSource: DataSource,
): Promise<any> {
const connectedAccounts = await workspaceDataSource?.query(
`SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "id" = $1`,
[connectedAccountId],
);
if (!connectedAccounts || connectedAccounts.length === 0) {
throw new Error('No connected account found');
}
return connectedAccounts;
return connectedAccounts[0];
}
public async getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
public async getDataSourceMetadataWorkspaceMetadata(
workspaceId: string,
connectedAccountId: string,
): Promise<{
dataSourceMetadata: DataSourceEntity;
workspaceDataSource: DataSource;
connectedAccount: any;
}> {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
@ -248,19 +248,9 @@ export class MessagingUtilsService {
throw new Error('No workspace data source found');
}
const connectedAccounts = await workspaceDataSource?.query(
`SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'google' AND "id" = $1`,
[connectedAccountId],
);
if (!connectedAccounts || connectedAccounts.length === 0) {
throw new Error('No connected account found');
}
return {
dataSourceMetadata,
workspaceDataSource,
connectedAccount: connectedAccounts[0],
};
}