6255 move services from messaging common module into the correct module and refactor them (#6409)
Closes #6255 - Move files from `messaging/common` into the correct module - Remove common module between calendar and messaging `calendar-messaging-participant-manager` - Update and fix massaging and calendar participant matching - Create `MatchParticipantModule` --------- Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
@ -0,0 +1 @@
|
||||
export const MESSAGING_THROTTLE_DURATION = 1000 * 60 * 1; // 1 minute
|
||||
@ -0,0 +1 @@
|
||||
export const MESSAGING_THROTTLE_MAX_ATTEMPTS = 4;
|
||||
@ -1,3 +1,4 @@
|
||||
import { HttpModule } from '@nestjs/axios';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
@ -9,24 +10,22 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
|
||||
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
|
||||
import { EmailAliasManagerModule } from 'src/modules/connected-account/email-alias-manager/email-alias-manager.module';
|
||||
import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
|
||||
import { RefreshAccessTokenManagerModule } from 'src/modules/connected-account/refresh-access-token-manager/refresh-access-token-manager.module';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
|
||||
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessagingGmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/messaging-gmail-client.provider';
|
||||
import { MessagingGmailFetchByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-by-batch.service';
|
||||
import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-by-batches.service';
|
||||
import { MessagingGmailFetchMessageIdsToExcludeService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-ids-to-exclude.service';
|
||||
import { MessagingGmailFullMessageListFetchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-full-message-list-fetch.service';
|
||||
import { MessagingGmailHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-history.service';
|
||||
import { MessagingGmailMessagesImportService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service';
|
||||
import { MessagingGmailPartialMessageListFetchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-partial-message-list-fetch.service';
|
||||
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-save-messages-and-enqueue-contact-creation.service';
|
||||
import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
RefreshAccessTokenManagerModule,
|
||||
HttpModule.register({
|
||||
baseURL: 'https://www.googleapis.com/batch/gmail/v1',
|
||||
}),
|
||||
EnvironmentModule,
|
||||
ObjectMetadataRepositoryModule.forFeature([
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
@ -45,20 +44,15 @@ import { MessageParticipantManagerModule } from 'src/modules/messaging/message-p
|
||||
providers: [
|
||||
MessagingGmailClientProvider,
|
||||
MessagingGmailHistoryService,
|
||||
MessagingGmailFetchByBatchesService,
|
||||
MessagingGmailFetchMessagesByBatchesService,
|
||||
MessagingGmailPartialMessageListFetchService,
|
||||
MessagingGmailFullMessageListFetchService,
|
||||
MessagingGmailMessagesImportService,
|
||||
MessagingGmailFetchMessageIdsToExcludeService,
|
||||
MessagingSaveMessagesAndEnqueueContactCreationService,
|
||||
],
|
||||
exports: [
|
||||
MessagingGmailClientProvider,
|
||||
MessagingGmailHistoryService,
|
||||
MessagingGmailFetchByBatchesService,
|
||||
MessagingGmailFetchMessagesByBatchesService,
|
||||
MessagingGmailPartialMessageListFetchService,
|
||||
MessagingGmailFullMessageListFetchService,
|
||||
MessagingGmailMessagesImportService,
|
||||
MessagingGmailFetchMessageIdsToExcludeService,
|
||||
],
|
||||
})
|
||||
|
||||
@ -0,0 +1,140 @@
|
||||
import { HttpService } from '@nestjs/axios';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { AxiosResponse } from 'axios';
|
||||
|
||||
import { GmailMessageParsedResponse } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message-parsed-response';
|
||||
import { BatchQueries } from 'src/modules/messaging/message-import-manager/types/batch-queries';
|
||||
import { createQueriesFromMessageIds } from 'src/modules/messaging/message-import-manager/utils/create-queries-from-message-ids.util';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingGmailFetchByBatchesService {
|
||||
constructor(private readonly httpService: HttpService) {}
|
||||
|
||||
async fetchAllByBatches(
|
||||
messageIds: string[],
|
||||
accessToken: string,
|
||||
boundary: string,
|
||||
): Promise<{
|
||||
messageIdsByBatch: string[][];
|
||||
batchResponses: AxiosResponse<any, any>[];
|
||||
}> {
|
||||
const batchLimit = 20;
|
||||
|
||||
let batchOffset = 0;
|
||||
|
||||
let batchResponses: AxiosResponse<any, any>[] = [];
|
||||
|
||||
const messageIdsByBatch: string[][] = [];
|
||||
|
||||
while (batchOffset < messageIds.length) {
|
||||
const batchResponse = await this.fetchBatch(
|
||||
messageIds,
|
||||
accessToken,
|
||||
batchOffset,
|
||||
batchLimit,
|
||||
boundary,
|
||||
);
|
||||
|
||||
batchResponses = batchResponses.concat(batchResponse);
|
||||
|
||||
messageIdsByBatch.push(
|
||||
messageIds.slice(batchOffset, batchOffset + batchLimit),
|
||||
);
|
||||
|
||||
batchOffset += batchLimit;
|
||||
}
|
||||
|
||||
return { messageIdsByBatch, batchResponses };
|
||||
}
|
||||
|
||||
async fetchBatch(
|
||||
messageIds: string[],
|
||||
accessToken: string,
|
||||
batchOffset: number,
|
||||
batchLimit: number,
|
||||
boundary: string,
|
||||
): Promise<AxiosResponse<any, any>> {
|
||||
const queries = createQueriesFromMessageIds(messageIds);
|
||||
|
||||
const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit);
|
||||
|
||||
const response = await this.httpService.axiosRef.post(
|
||||
'/',
|
||||
this.createBatchBody(limitedQueries, boundary),
|
||||
{
|
||||
headers: {
|
||||
'Content-Type': 'multipart/mixed; boundary=' + boundary,
|
||||
Authorization: 'Bearer ' + accessToken,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
createBatchBody(queries: BatchQueries, boundary: string): string {
|
||||
let batchBody: string[] = [];
|
||||
|
||||
queries.forEach(function (call) {
|
||||
const method = 'GET';
|
||||
const uri = call.uri;
|
||||
|
||||
batchBody = batchBody.concat([
|
||||
'--',
|
||||
boundary,
|
||||
'\r\n',
|
||||
'Content-Type: application/http',
|
||||
'\r\n\r\n',
|
||||
|
||||
method,
|
||||
' ',
|
||||
uri,
|
||||
'\r\n\r\n',
|
||||
]);
|
||||
});
|
||||
|
||||
return batchBody.concat(['--', boundary, '--']).join('');
|
||||
}
|
||||
|
||||
parseBatch(
|
||||
responseCollection: AxiosResponse<any, any>,
|
||||
): GmailMessageParsedResponse[] {
|
||||
const responseItems: GmailMessageParsedResponse[] = [];
|
||||
|
||||
const boundary = this.getBatchSeparator(responseCollection);
|
||||
|
||||
const responseLines: string[] = responseCollection.data.split(
|
||||
'--' + boundary,
|
||||
);
|
||||
|
||||
responseLines.forEach(function (response) {
|
||||
const startJson = response.indexOf('{');
|
||||
const endJson = response.lastIndexOf('}');
|
||||
|
||||
if (startJson < 0 || endJson < 0) return;
|
||||
|
||||
const responseJson = response.substring(startJson, endJson + 1);
|
||||
|
||||
const item = JSON.parse(responseJson);
|
||||
|
||||
responseItems.push(item);
|
||||
});
|
||||
|
||||
return responseItems;
|
||||
}
|
||||
|
||||
getBatchSeparator(responseCollection: AxiosResponse<any, any>): string {
|
||||
const headers = responseCollection.headers;
|
||||
|
||||
const contentType: string = headers['content-type'];
|
||||
|
||||
if (!contentType) return '';
|
||||
|
||||
const components = contentType.split('; ');
|
||||
|
||||
const boundary = components.find((item) => item.startsWith('boundary='));
|
||||
|
||||
return boundary?.replace('boundary=', '').trim() || '';
|
||||
}
|
||||
}
|
||||
@ -1,14 +1,14 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { AxiosResponse } from 'axios';
|
||||
import planer from 'planer';
|
||||
import addressparser from 'addressparser';
|
||||
import { AxiosResponse } from 'axios';
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import planer from 'planer';
|
||||
|
||||
import { assert, assertNotNull } from 'src/utils/assert';
|
||||
import { MessagingGmailFetchByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-by-batch.service';
|
||||
import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
|
||||
import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util';
|
||||
import { MessagingFetchByBatchesService } from 'src/modules/messaging/common/services/messaging-fetch-by-batch.service';
|
||||
import { assert, assertNotNull } from 'src/utils/assert';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingGmailFetchMessagesByBatchesService {
|
||||
@ -17,7 +17,7 @@ export class MessagingGmailFetchMessagesByBatchesService {
|
||||
);
|
||||
|
||||
constructor(
|
||||
private readonly fetchByBatchesService: MessagingFetchByBatchesService,
|
||||
private readonly fetchByBatchesService: MessagingGmailFetchByBatchesService,
|
||||
) {}
|
||||
|
||||
async fetchAllMessages(
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import { GaxiosResponse } from 'gaxios';
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
|
||||
import { GmailError } from 'src/modules/messaging/common/services/messaging-error-handling.service';
|
||||
import { MESSAGING_GMAIL_USERS_HISTORY_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-history-max-result.constant';
|
||||
import { GmailError } from 'src/modules/messaging/message-import-manager/services/messaging-error-handling.service';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingGmailHistoryService {
|
||||
|
||||
@ -6,15 +6,15 @@ import { MessageQueue } from 'src/engine/integrations/message-queue/message-queu
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
|
||||
import { MessagingTelemetryService } from 'src/modules/messaging/common/services/messaging-telemetry.service';
|
||||
import {
|
||||
MessageChannelSyncStage,
|
||||
MessageChannelWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessagingGmailFullMessageListFetchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-full-message-list-fetch.service';
|
||||
import { MessagingGmailPartialMessageListFetchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-partial-message-list-fetch.service';
|
||||
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
|
||||
import { MessagingFullMessageListFetchService } from 'src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service';
|
||||
import { MessagingPartialMessageListFetchService } from 'src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service';
|
||||
import { MessagingTelemetryService } from 'src/modules/messaging/monitoring/services/messaging-telemetry.service';
|
||||
|
||||
export type MessagingMessageListFetchJobData = {
|
||||
messageChannelId: string;
|
||||
@ -29,8 +29,8 @@ export class MessagingMessageListFetchJob {
|
||||
private readonly logger = new Logger(MessagingMessageListFetchJob.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailFullMessageListFetchService: MessagingGmailFullMessageListFetchService,
|
||||
private readonly gmailPartialMessageListFetchV2Service: MessagingGmailPartialMessageListFetchService,
|
||||
private readonly messagingFullMessageListFetchService: MessagingFullMessageListFetchService,
|
||||
private readonly messagingPartialMessageListFetchService: MessagingPartialMessageListFetchService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
|
||||
@ -95,7 +95,7 @@ export class MessagingMessageListFetchJob {
|
||||
messageChannelId: messageChannel.id,
|
||||
});
|
||||
|
||||
await this.gmailPartialMessageListFetchV2Service.processMessageListFetch(
|
||||
await this.messagingPartialMessageListFetchService.processMessageListFetch(
|
||||
messageChannel,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
@ -122,7 +122,7 @@ export class MessagingMessageListFetchJob {
|
||||
messageChannelId: messageChannel.id,
|
||||
});
|
||||
|
||||
await this.gmailFullMessageListFetchService.processMessageListFetch(
|
||||
await this.messagingFullMessageListFetchService.processMessageListFetch(
|
||||
messageChannel,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
|
||||
@ -6,14 +6,14 @@ import { MessageQueue } from 'src/engine/integrations/message-queue/message-queu
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
|
||||
import { MessagingTelemetryService } from 'src/modules/messaging/common/services/messaging-telemetry.service';
|
||||
import {
|
||||
MessageChannelSyncStage,
|
||||
MessageChannelWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessagingGmailMessagesImportService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service';
|
||||
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
|
||||
import { MessagingMessagesImportService } from 'src/modules/messaging/message-import-manager/services/messaging-messages-import.service';
|
||||
import { MessagingTelemetryService } from 'src/modules/messaging/monitoring/services/messaging-telemetry.service';
|
||||
|
||||
export type MessagingMessagesImportJobData = {
|
||||
messageChannelId: string;
|
||||
@ -28,7 +28,7 @@ export class MessagingMessagesImportJob {
|
||||
constructor(
|
||||
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
private readonly gmailFetchMessageContentFromCacheService: MessagingGmailMessagesImportService,
|
||||
private readonly gmailFetchMessageContentFromCacheService: MessagingMessagesImportService,
|
||||
@InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
private readonly messagingTelemetryService: MessagingTelemetryService,
|
||||
|
||||
@ -1,9 +1,14 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
|
||||
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||
import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { EmailAliasManagerModule } from 'src/modules/connected-account/email-alias-manager/email-alias-manager.module';
|
||||
import { RefreshAccessTokenManagerModule } from 'src/modules/connected-account/refresh-access-token-manager/refresh-access-token-manager.module';
|
||||
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessagingSingleMessageImportCommand } from 'src/modules/messaging/message-import-manager/commands/messaging-single-message-import.command';
|
||||
@ -20,11 +25,19 @@ import { MessagingMessageListFetchJob } from 'src/modules/messaging/message-impo
|
||||
import { MessagingMessagesImportJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job';
|
||||
import { MessagingOngoingStaleJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job';
|
||||
import { MessagingMessageImportManagerMessageChannelListener } from 'src/modules/messaging/message-import-manager/listeners/messaging-import-manager-message-channel.listener';
|
||||
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { MessagingErrorHandlingService } from 'src/modules/messaging/message-import-manager/services/messaging-error-handling.service';
|
||||
import { MessagingFullMessageListFetchService } from 'src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service';
|
||||
import { MessagingMessageThreadService } from 'src/modules/messaging/message-import-manager/services/messaging-message-thread.service';
|
||||
import { MessagingMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-message.service';
|
||||
import { MessagingMessagesImportService } from 'src/modules/messaging/message-import-manager/services/messaging-messages-import.service';
|
||||
import { MessagingPartialMessageListFetchService } from 'src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service';
|
||||
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service';
|
||||
import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module';
|
||||
import { MessagingMonitoringModule } from 'src/modules/messaging/monitoring/messaging-monitoring.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
RefreshAccessTokenManagerModule,
|
||||
WorkspaceDataSourceModule,
|
||||
MessagingGmailDriverModule,
|
||||
MessagingCommonModule,
|
||||
@ -32,6 +45,10 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
|
||||
TypeOrmModule.forFeature([DataSourceEntity], 'metadata'),
|
||||
TwentyORMModule.forFeature([MessageChannelWorkspaceEntity]),
|
||||
BillingModule,
|
||||
EmailAliasManagerModule,
|
||||
FeatureFlagModule,
|
||||
MessageParticipantManagerModule,
|
||||
MessagingMonitoringModule,
|
||||
],
|
||||
providers: [
|
||||
MessagingMessageListFetchCronCommand,
|
||||
@ -47,6 +64,13 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
|
||||
MessagingAddSingleMessageToCacheForImportJob,
|
||||
MessagingMessageImportManagerMessageChannelListener,
|
||||
MessagingCleanCacheJob,
|
||||
MessagingMessageService,
|
||||
MessagingMessageThreadService,
|
||||
MessagingErrorHandlingService,
|
||||
MessagingPartialMessageListFetchService,
|
||||
MessagingFullMessageListFetchService,
|
||||
MessagingMessagesImportService,
|
||||
MessagingSaveMessagesAndEnqueueContactCreationService,
|
||||
],
|
||||
exports: [],
|
||||
})
|
||||
|
||||
@ -0,0 +1,330 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import snakeCase from 'lodash.snakecase';
|
||||
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
|
||||
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MESSAGING_THROTTLE_MAX_ATTEMPTS } from 'src/modules/messaging/message-import-manager/constants/messaging-throttle-max-attempts';
|
||||
import { MessagingTelemetryService } from 'src/modules/messaging/monitoring/services/messaging-telemetry.service';
|
||||
|
||||
type SyncStep =
|
||||
| 'partial-message-list-fetch'
|
||||
| 'full-message-list-fetch'
|
||||
| 'messages-import';
|
||||
|
||||
export type GmailError = {
|
||||
code: number | string;
|
||||
reason: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class MessagingErrorHandlingService {
|
||||
constructor(
|
||||
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService,
|
||||
private readonly messagingTelemetryService: MessagingTelemetryService,
|
||||
@InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
) {}
|
||||
|
||||
public async handleGmailError(
|
||||
error: GmailError,
|
||||
syncStep: SyncStep,
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
const { code, reason } = error;
|
||||
|
||||
switch (code) {
|
||||
case 400:
|
||||
if (reason === 'invalid_grant') {
|
||||
await this.handleInsufficientPermissions(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
if (reason === 'failedPrecondition') {
|
||||
await this.handleFailedPrecondition(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
} else {
|
||||
await this.handleUnknownError(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
break;
|
||||
case 404:
|
||||
await this.handleNotFound(error, syncStep, messageChannel, workspaceId);
|
||||
break;
|
||||
|
||||
case 429:
|
||||
await this.handleRateLimitExceeded(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
break;
|
||||
|
||||
case 403:
|
||||
if (
|
||||
reason === 'rateLimitExceeded' ||
|
||||
reason === 'userRateLimitExceeded'
|
||||
) {
|
||||
await this.handleRateLimitExceeded(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
} else {
|
||||
await this.handleInsufficientPermissions(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
break;
|
||||
|
||||
case 401:
|
||||
await this.handleInsufficientPermissions(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
break;
|
||||
case 500:
|
||||
if (reason === 'backendError') {
|
||||
await this.handleRateLimitExceeded(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
} else {
|
||||
await this.messagingChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
throw new Error(
|
||||
`Unhandled Gmail error code ${code} with reason ${reason}`,
|
||||
);
|
||||
}
|
||||
break;
|
||||
case 'ECONNRESET':
|
||||
case 'ENOTFOUND':
|
||||
case 'ECONNABORTED':
|
||||
case 'ETIMEDOUT':
|
||||
case 'ERR_NETWORK':
|
||||
// We are currently mixing up Gmail Error code (HTTP status) and axios error code (ECONNRESET)
|
||||
|
||||
// In case of a network error, we should retry the request
|
||||
await this.handleRateLimitExceeded(
|
||||
error,
|
||||
syncStep,
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
);
|
||||
break;
|
||||
default:
|
||||
await this.messagingChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
throw new Error(
|
||||
`Unhandled Gmail error code ${code} with reason ${reason}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async handleRateLimitExceeded(
|
||||
error: GmailError,
|
||||
syncStep: SyncStep,
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
await this.messagingTelemetryService.track({
|
||||
eventName: `${snakeCase(syncStep)}.error.rate_limit_exceeded`,
|
||||
workspaceId,
|
||||
connectedAccountId: messageChannel.connectedAccountId,
|
||||
messageChannelId: messageChannel.id,
|
||||
message: `${error.code}: ${error.reason}`,
|
||||
});
|
||||
|
||||
await this.handleThrottle(syncStep, messageChannel, workspaceId);
|
||||
}
|
||||
|
||||
private async handleFailedPrecondition(
|
||||
error: GmailError,
|
||||
syncStep: SyncStep,
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
await this.messagingTelemetryService.track({
|
||||
eventName: `${snakeCase(syncStep)}.error.failed_precondition`,
|
||||
workspaceId,
|
||||
connectedAccountId: messageChannel.connectedAccountId,
|
||||
messageChannelId: messageChannel.id,
|
||||
message: `${error.code}: ${error.reason}`,
|
||||
});
|
||||
|
||||
await this.handleThrottle(syncStep, messageChannel, workspaceId);
|
||||
}
|
||||
|
||||
private async handleInsufficientPermissions(
|
||||
error: GmailError,
|
||||
syncStep: SyncStep,
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
await this.messagingTelemetryService.track({
|
||||
eventName: `${snakeCase(syncStep)}.error.insufficient_permissions`,
|
||||
workspaceId,
|
||||
connectedAccountId: messageChannel.connectedAccountId,
|
||||
messageChannelId: messageChannel.id,
|
||||
message: `${error.code}: ${error.reason}`,
|
||||
});
|
||||
|
||||
await this.messagingChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushMessagesToImport(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!messageChannel.connectedAccountId) {
|
||||
throw new Error(
|
||||
`Connected account ID is not defined for message channel ${messageChannel.id} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
await this.connectedAccountRepository.updateAuthFailedAt(
|
||||
messageChannel.connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
private async handleNotFound(
|
||||
error: GmailError,
|
||||
syncStep: SyncStep,
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
if (syncStep === 'messages-import') {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.messagingTelemetryService.track({
|
||||
eventName: `${snakeCase(syncStep)}.error.not_found`,
|
||||
workspaceId,
|
||||
connectedAccountId: messageChannel.connectedAccountId,
|
||||
messageChannelId: messageChannel.id,
|
||||
message: `404: ${error.reason}`,
|
||||
});
|
||||
|
||||
await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
private async handleThrottle(
|
||||
syncStep: SyncStep,
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
if (
|
||||
messageChannel.throttleFailureCount >= MESSAGING_THROTTLE_MAX_ATTEMPTS
|
||||
) {
|
||||
await this.messagingChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.throttle(messageChannel, workspaceId);
|
||||
|
||||
switch (syncStep) {
|
||||
case 'full-message-list-fetch':
|
||||
await this.messagingChannelSyncStatusService.scheduleFullMessageListFetch(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
break;
|
||||
|
||||
case 'partial-message-list-fetch':
|
||||
await this.messagingChannelSyncStatusService.schedulePartialMessageListFetch(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
break;
|
||||
|
||||
case 'messages-import':
|
||||
await this.messagingChannelSyncStatusService.scheduleMessagesImport(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private async throttle(
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
await this.messageChannelRepository.incrementThrottleFailureCount(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.messagingTelemetryService.track({
|
||||
eventName: 'message_channel.throttle',
|
||||
workspaceId,
|
||||
connectedAccountId: messageChannel.connectedAccountId,
|
||||
messageChannelId: messageChannel.id,
|
||||
message: `Increment throttle failure count to ${messageChannel.throttleFailureCount}`,
|
||||
});
|
||||
}
|
||||
|
||||
private async handleUnknownError(
|
||||
error: GmailError,
|
||||
syncStep: SyncStep,
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
await this.messagingTelemetryService.track({
|
||||
eventName: `${snakeCase(syncStep)}.error.unknown`,
|
||||
workspaceId,
|
||||
connectedAccountId: messageChannel.connectedAccountId,
|
||||
messageChannelId: messageChannel.id,
|
||||
message: `${error.code}: ${error.reason}`,
|
||||
});
|
||||
|
||||
await this.messagingChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Unhandled Gmail error code ${error.code} with reason ${error.reason}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,8 +1,8 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import { GaxiosResponse } from 'gaxios';
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
@ -11,22 +11,22 @@ import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repos
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/common/repositories/message-channel-message-association.repository';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
|
||||
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
|
||||
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories';
|
||||
import { MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-list-max-result.constant';
|
||||
import { MessagingGmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/messaging-gmail-client.provider';
|
||||
import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter';
|
||||
import {
|
||||
GmailError,
|
||||
MessagingErrorHandlingService,
|
||||
} from 'src/modules/messaging/common/services/messaging-error-handling.service';
|
||||
import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter';
|
||||
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
|
||||
import { MessagingGmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/messaging-gmail-client.provider';
|
||||
import { MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-list-max-result.constant';
|
||||
import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories';
|
||||
} from 'src/modules/messaging/message-import-manager/services/messaging-error-handling.service';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingGmailFullMessageListFetchService {
|
||||
export class MessagingFullMessageListFetchService {
|
||||
private readonly logger = new Logger(
|
||||
MessagingGmailFullMessageListFetchService.name,
|
||||
MessagingFullMessageListFetchService.name,
|
||||
);
|
||||
|
||||
constructor(
|
||||
@ -56,12 +56,11 @@ export class MessagingGmailFullMessageListFetchService {
|
||||
const gmailClient: gmail_v1.Gmail =
|
||||
await this.gmailClientProvider.getGmailClient(connectedAccount);
|
||||
|
||||
const { error: gmailError } =
|
||||
await this.fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||
gmailClient,
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
const { error: gmailError } = await this.fetchAllMessageIdsAndStoreInCache(
|
||||
gmailClient,
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (gmailError) {
|
||||
await this.gmailErrorHandlingService.handleGmailError(
|
||||
@ -90,7 +89,7 @@ export class MessagingGmailFullMessageListFetchService {
|
||||
);
|
||||
}
|
||||
|
||||
private async fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||
private async fetchAllMessageIdsAndStoreInCache(
|
||||
gmailClient: gmail_v1.Gmail,
|
||||
messageChannelId: string,
|
||||
workspaceId: string,
|
||||
@ -0,0 +1,73 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/common/repositories/message-channel-message-association.repository';
|
||||
import { MessageThreadRepository } from 'src/modules/messaging/common/repositories/message-thread.repository';
|
||||
import { MessageRepository } from 'src/modules/messaging/common/repositories/message.repository';
|
||||
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
import { MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
|
||||
import { MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingMessageThreadService {
|
||||
constructor(
|
||||
@InjectObjectMetadataRepository(
|
||||
MessageChannelMessageAssociationWorkspaceEntity,
|
||||
)
|
||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||
@InjectObjectMetadataRepository(MessageWorkspaceEntity)
|
||||
private readonly messageRepository: MessageRepository,
|
||||
@InjectObjectMetadataRepository(MessageThreadWorkspaceEntity)
|
||||
private readonly messageThreadRepository: MessageThreadRepository,
|
||||
) {}
|
||||
|
||||
public async saveMessageThreadOrReturnExistingMessageThread(
|
||||
headerMessageId: string,
|
||||
messageThreadExternalId: string,
|
||||
workspaceId: string,
|
||||
manager: EntityManager,
|
||||
) {
|
||||
// Check if message thread already exists via threadExternalId
|
||||
const existingMessageChannelMessageAssociationByMessageThreadExternalId =
|
||||
await this.messageChannelMessageAssociationRepository.getFirstByMessageThreadExternalId(
|
||||
messageThreadExternalId,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
const existingMessageThread =
|
||||
existingMessageChannelMessageAssociationByMessageThreadExternalId?.messageThreadId;
|
||||
|
||||
if (existingMessageThread) {
|
||||
return Promise.resolve(existingMessageThread);
|
||||
}
|
||||
|
||||
// Check if message thread already exists via existing message headerMessageId
|
||||
const existingMessageWithSameHeaderMessageId =
|
||||
await this.messageRepository.getFirstOrNullByHeaderMessageId(
|
||||
headerMessageId,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
if (existingMessageWithSameHeaderMessageId) {
|
||||
return Promise.resolve(
|
||||
existingMessageWithSameHeaderMessageId.messageThreadId,
|
||||
);
|
||||
}
|
||||
|
||||
// If message thread does not exist, create new message thread
|
||||
const newMessageThreadId = v4();
|
||||
|
||||
await this.messageThreadRepository.insert(
|
||||
newMessageThreadId,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
return Promise.resolve(newMessageThreadId);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,233 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/common/repositories/message-channel-message-association.repository';
|
||||
import { MessageThreadRepository } from 'src/modules/messaging/common/repositories/message-thread.repository';
|
||||
import { MessageRepository } from 'src/modules/messaging/common/repositories/message.repository';
|
||||
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
import { MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
|
||||
import { MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
|
||||
import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
|
||||
import { MessagingMessageThreadService } from 'src/modules/messaging/message-import-manager/services/messaging-message-thread.service';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingMessageService {
|
||||
constructor(
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
@InjectObjectMetadataRepository(
|
||||
MessageChannelMessageAssociationWorkspaceEntity,
|
||||
)
|
||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||
@InjectObjectMetadataRepository(MessageWorkspaceEntity)
|
||||
private readonly messageRepository: MessageRepository,
|
||||
@InjectObjectMetadataRepository(MessageThreadWorkspaceEntity)
|
||||
private readonly messageThreadRepository: MessageThreadRepository,
|
||||
private readonly messageThreadService: MessagingMessageThreadService,
|
||||
) {}
|
||||
|
||||
public async saveMessagesWithinTransaction(
|
||||
messages: GmailMessage[],
|
||||
connectedAccount: ConnectedAccountWorkspaceEntity,
|
||||
gmailMessageChannelId: string,
|
||||
workspaceId: string,
|
||||
transactionManager: EntityManager,
|
||||
): Promise<Map<string, string>> {
|
||||
const messageExternalIdsAndIdsMap = new Map<string, string>();
|
||||
|
||||
for (const message of messages) {
|
||||
const existingMessageChannelMessageAssociationsCount =
|
||||
await this.messageChannelMessageAssociationRepository.countByMessageExternalIdsAndMessageChannelId(
|
||||
[message.externalId],
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (existingMessageChannelMessageAssociationsCount > 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: This does not handle all thread merging use cases and might create orphan threads.
|
||||
const savedOrExistingMessageThreadId =
|
||||
await this.messageThreadService.saveMessageThreadOrReturnExistingMessageThread(
|
||||
message.headerMessageId,
|
||||
message.messageThreadExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (!savedOrExistingMessageThreadId) {
|
||||
throw new Error(
|
||||
`No message thread found for message ${message.headerMessageId} in workspace ${workspaceId} in saveMessages`,
|
||||
);
|
||||
}
|
||||
|
||||
const savedOrExistingMessageId =
|
||||
await this.saveMessageOrReturnExistingMessage(
|
||||
message,
|
||||
savedOrExistingMessageThreadId,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
messageExternalIdsAndIdsMap.set(
|
||||
message.externalId,
|
||||
savedOrExistingMessageId,
|
||||
);
|
||||
|
||||
await this.messageChannelMessageAssociationRepository.insert(
|
||||
gmailMessageChannelId,
|
||||
savedOrExistingMessageId,
|
||||
message.externalId,
|
||||
savedOrExistingMessageThreadId,
|
||||
message.messageThreadExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
return messageExternalIdsAndIdsMap;
|
||||
}
|
||||
|
||||
private async saveMessageOrReturnExistingMessage(
|
||||
message: GmailMessage,
|
||||
messageThreadId: string,
|
||||
connectedAccount: ConnectedAccountWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
manager: EntityManager,
|
||||
): Promise<string> {
|
||||
const existingMessage =
|
||||
await this.messageRepository.getFirstOrNullByHeaderMessageId(
|
||||
message.headerMessageId,
|
||||
workspaceId,
|
||||
);
|
||||
const existingMessageId = existingMessage?.id;
|
||||
|
||||
if (existingMessageId) {
|
||||
return Promise.resolve(existingMessageId);
|
||||
}
|
||||
|
||||
const newMessageId = v4();
|
||||
|
||||
const messageDirection =
|
||||
connectedAccount.handle === message.fromHandle ||
|
||||
connectedAccount.handleAliases?.includes(message.fromHandle)
|
||||
? 'outgoing'
|
||||
: 'incoming';
|
||||
|
||||
const receivedAt = new Date(parseInt(message.internalDate));
|
||||
|
||||
await this.messageRepository.insert(
|
||||
newMessageId,
|
||||
message.headerMessageId,
|
||||
message.subject,
|
||||
receivedAt,
|
||||
messageDirection,
|
||||
messageThreadId,
|
||||
message.text,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
return Promise.resolve(newMessageId);
|
||||
}
|
||||
|
||||
public async deleteMessages(
|
||||
messagesDeletedMessageExternalIds: string[],
|
||||
gmailMessageChannelId: string,
|
||||
workspaceId: string,
|
||||
) {
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await workspaceDataSource?.transaction(async (manager: EntityManager) => {
|
||||
const messageChannelMessageAssociationsToDelete =
|
||||
await this.messageChannelMessageAssociationRepository.getByMessageExternalIdsAndMessageChannelId(
|
||||
messagesDeletedMessageExternalIds,
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
const messageChannelMessageAssociationIdsToDeleteIds =
|
||||
messageChannelMessageAssociationsToDelete.map(
|
||||
(messageChannelMessageAssociationToDelete) =>
|
||||
messageChannelMessageAssociationToDelete.id,
|
||||
);
|
||||
|
||||
await this.messageChannelMessageAssociationRepository.deleteByIds(
|
||||
messageChannelMessageAssociationIdsToDeleteIds,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
const messageIdsFromMessageChannelMessageAssociationsToDelete =
|
||||
messageChannelMessageAssociationsToDelete.map(
|
||||
(messageChannelMessageAssociationToDelete) =>
|
||||
messageChannelMessageAssociationToDelete.messageId,
|
||||
);
|
||||
|
||||
const messageChannelMessageAssociationByMessageIds =
|
||||
await this.messageChannelMessageAssociationRepository.getByMessageIds(
|
||||
messageIdsFromMessageChannelMessageAssociationsToDelete,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
const messageIdsFromMessageChannelMessageAssociationByMessageIds =
|
||||
messageChannelMessageAssociationByMessageIds.map(
|
||||
(messageChannelMessageAssociation) =>
|
||||
messageChannelMessageAssociation.messageId,
|
||||
);
|
||||
|
||||
const messageIdsToDelete =
|
||||
messageIdsFromMessageChannelMessageAssociationsToDelete.filter(
|
||||
(messageId) =>
|
||||
!messageIdsFromMessageChannelMessageAssociationByMessageIds.includes(
|
||||
messageId,
|
||||
),
|
||||
);
|
||||
|
||||
await this.messageRepository.deleteByIds(
|
||||
messageIdsToDelete,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
const messageThreadIdsFromMessageChannelMessageAssociationsToDelete =
|
||||
messageChannelMessageAssociationsToDelete.map(
|
||||
(messageChannelMessageAssociationToDelete) =>
|
||||
messageChannelMessageAssociationToDelete.messageThreadId,
|
||||
);
|
||||
|
||||
const messagesByThreadIds =
|
||||
await this.messageRepository.getByMessageThreadIds(
|
||||
messageThreadIdsFromMessageChannelMessageAssociationsToDelete,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
|
||||
const threadIdsToDelete =
|
||||
messageThreadIdsFromMessageChannelMessageAssociationsToDelete.filter(
|
||||
(threadId) =>
|
||||
!messagesByThreadIds.find(
|
||||
(message) => message.messageThreadId === threadId,
|
||||
),
|
||||
);
|
||||
|
||||
await this.messageThreadRepository.deleteByIds(
|
||||
threadIdsToDelete,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -14,22 +14,20 @@ import { ConnectedAccountRepository } from 'src/modules/connected-account/reposi
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
|
||||
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
|
||||
import { MessagingErrorHandlingService } from 'src/modules/messaging/common/services/messaging-error-handling.service';
|
||||
import { MessagingTelemetryService } from 'src/modules/messaging/common/services/messaging-telemetry.service';
|
||||
import {
|
||||
MessageChannelSyncStage,
|
||||
MessageChannelWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';
|
||||
import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-by-batches.service';
|
||||
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-save-messages-and-enqueue-contact-creation.service';
|
||||
import { MessagingErrorHandlingService } from 'src/modules/messaging/message-import-manager/services/messaging-error-handling.service';
|
||||
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service';
|
||||
import { filterEmails } from 'src/modules/messaging/message-import-manager/utils/filter-emails.util';
|
||||
import { MessagingTelemetryService } from 'src/modules/messaging/monitoring/services/messaging-telemetry.service';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingGmailMessagesImportService {
|
||||
private readonly logger = new Logger(
|
||||
MessagingGmailMessagesImportService.name,
|
||||
);
|
||||
export class MessagingMessagesImportService {
|
||||
private readonly logger = new Logger(MessagingMessagesImportService.name);
|
||||
|
||||
constructor(
|
||||
private readonly fetchMessagesByBatchesService: MessagingGmailFetchMessagesByBatchesService,
|
||||
@ -2,25 +2,25 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/common/repositories/message-channel-message-association.repository';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
|
||||
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
|
||||
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessagingGmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/messaging-gmail-client.provider';
|
||||
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
|
||||
import { MessagingGmailFetchMessageIdsToExcludeService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-ids-to-exclude.service';
|
||||
import { MessagingErrorHandlingService } from 'src/modules/messaging/common/services/messaging-error-handling.service';
|
||||
import { MessagingGmailHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-history.service';
|
||||
import { MessagingErrorHandlingService } from 'src/modules/messaging/message-import-manager/services/messaging-error-handling.service';
|
||||
|
||||
@Injectable()
|
||||
export class MessagingGmailPartialMessageListFetchService {
|
||||
export class MessagingPartialMessageListFetchService {
|
||||
private readonly logger = new Logger(
|
||||
MessagingGmailPartialMessageListFetchService.name,
|
||||
MessagingPartialMessageListFetchService.name,
|
||||
);
|
||||
|
||||
constructor(
|
||||
@ -1,30 +1,26 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { EntityManager, Repository } from 'typeorm';
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import {
|
||||
CreateCompanyAndContactJob,
|
||||
CreateCompanyAndContactJobData,
|
||||
} from 'src/modules/contact-creation-manager/jobs/create-company-and-contact.job';
|
||||
import { MessagingMessageService } from 'src/modules/messaging/common/services/messaging-message.service';
|
||||
import {
|
||||
MessageChannelContactAutoCreationPolicy,
|
||||
MessageChannelWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
|
||||
import {
|
||||
GmailMessage,
|
||||
Participant,
|
||||
ParticipantWithMessageId,
|
||||
} from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
|
||||
import { MessagingMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-message.service';
|
||||
import { MessagingMessageParticipantService } from 'src/modules/messaging/message-participant-manager/services/messaging-message-participant.service';
|
||||
import { isGroupEmail } from 'src/utils/is-group-email';
|
||||
import { isWorkEmail } from 'src/utils/is-work-email';
|
||||
@ -32,14 +28,11 @@ import { isWorkEmail } from 'src/utils/is-work-email';
|
||||
@Injectable()
|
||||
export class MessagingSaveMessagesAndEnqueueContactCreationService {
|
||||
constructor(
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
@InjectMessageQueue(MessageQueue.contactCreationQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
private readonly messageService: MessagingMessageService,
|
||||
private readonly messageParticipantService: MessagingMessageParticipantService,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
private readonly eventEmitter: EventEmitter2,
|
||||
private readonly twentyORMManager: TwentyORMManager,
|
||||
) {}
|
||||
|
||||
async saveMessagesAndEnqueueContactCreationJob(
|
||||
@ -48,14 +41,9 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
|
||||
connectedAccount: ConnectedAccountWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
) {
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const handleAliases = connectedAccount.handleAliases?.split(',') || [];
|
||||
|
||||
let savedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
|
||||
const workspaceDataSource = await this.twentyORMManager.getDatasource();
|
||||
|
||||
const participantsWithMessageId = await workspaceDataSource?.transaction(
|
||||
async (transactionManager: EntityManager) => {
|
||||
@ -114,23 +102,15 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
|
||||
: [];
|
||||
});
|
||||
|
||||
savedMessageParticipants =
|
||||
await this.messageParticipantService.saveMessageParticipants(
|
||||
participantsWithMessageId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
await this.messageParticipantService.saveMessageParticipants(
|
||||
participantsWithMessageId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
return participantsWithMessageId;
|
||||
},
|
||||
);
|
||||
|
||||
this.eventEmitter.emit(`messageParticipant.matched`, {
|
||||
workspaceId,
|
||||
workspaceMemberId: connectedAccount.accountOwnerId,
|
||||
messageParticipants: savedMessageParticipants,
|
||||
});
|
||||
|
||||
if (messageChannel.isContactAutoCreationEnabled) {
|
||||
const contactsToCreate = participantsWithMessageId.filter(
|
||||
(participant) => participant.shouldCreateContact,
|
||||
@ -1,4 +1,4 @@
|
||||
import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant-manager/utils/is-email-blocklisted.util';
|
||||
import { isEmailBlocklisted } from 'src/modules/blocklist/utils/is-email-blocklisted.util';
|
||||
import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
|
||||
|
||||
// Todo: refactor this into several utils
|
||||
|
||||
Reference in New Issue
Block a user