admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor calendar to use new sync statuses and stages (#6141)

Browse Source 
- Refactor calendar modules and some messaging modules to better
organize them by business rules and decouple them
- Work toward a common architecture for the different calendar providers
by introducing interfaces for the drivers
- Modify cron job to use the new sync statuses and stages
This commit is contained in:
bosiraphael
2024-07-08 17:01:06 +02:00
committed by GitHub
parent 1c3ea9b106
commit f458322303
69 changed files with 1300 additions and 884 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module.ts
View File

@ -5,6 +5,7 @@ import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { EmailAliasManagerModule } from 'src/modules/connected-account/email-alias-manager/email-alias-manager.module';
import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
import { GoogleAPIRefreshAccessTokenModule } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.module';
@ -20,6 +21,8 @@ import { MessagingGmailFullMessageListFetchService } from 'src/modules/messaging
import { MessagingGmailHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-history.service';
import { MessagingGmailMessagesImportService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service';
import { MessagingGmailPartialMessageListFetchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-partial-message-list-fetch.service';
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-save-messages-and-enqueue-contact-creation.service';
import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module';
@Module({
imports: [
@ -36,6 +39,8 @@ import { MessagingGmailPartialMessageListFetchService } from 'src/modules/messag
OAuth2ClientManagerModule,
EmailAliasManagerModule,
FeatureFlagModule,
WorkspaceDataSourceModule,
MessageParticipantManagerModule,
],
providers: [
MessagingGmailClientProvider,
@ -45,6 +50,7 @@ import { MessagingGmailPartialMessageListFetchService } from 'src/modules/messag
MessagingGmailFullMessageListFetchService,
MessagingGmailMessagesImportService,
MessagingGmailFetchMessageIdsToExcludeService,
MessagingSaveMessagesAndEnqueueContactCreationService,
],
exports: [
MessagingGmailClientProvider,

5
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/providers/messaging-gmail-client.provider.ts
View File

@ -12,7 +12,10 @@ export class MessagingGmailClientProvider {
) {}
public async getGmailClient(
connectedAccount: ConnectedAccountWorkspaceEntity,
connectedAccount: Pick<
ConnectedAccountWorkspaceEntity,
'provider' | 'refreshToken'
>,
): Promise<gmail_v1.Gmail> {
const oAuth2Client =
await this.oAuth2ClientManagerService.getOAuth2Client(connectedAccount);

2
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts
View File

@ -15,7 +15,6 @@ import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/s
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
import { MessagingErrorHandlingService } from 'src/modules/messaging/common/services/messaging-error-handling.service';
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/common/services/messaging-save-messages-and-enqueue-contact-creation.service';
import { MessagingTelemetryService } from 'src/modules/messaging/common/services/messaging-telemetry.service';
import {
MessageChannelSyncStage,
@ -23,6 +22,7 @@ import {
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';
import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-by-batches.service';
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-save-messages-and-enqueue-contact-creation.service';
import { filterEmails } from 'src/modules/messaging/message-import-manager/utils/filter-emails.util';
@Injectable()

148
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-save-messages-and-enqueue-contact-creation.service.ts Normal file
View File

@ -0,0 +1,148 @@
import { Injectable } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { InjectRepository } from '@nestjs/typeorm';
import { EntityManager, Repository } from 'typeorm';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import {
CreateCompanyAndContactJob,
CreateCompanyAndContactJobData,
} from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { MessagingMessageService } from 'src/modules/messaging/common/services/messaging-message.service';
import {
MessageChannelContactAutoCreationPolicy,
MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
import {
GmailMessage,
Participant,
ParticipantWithMessageId,
} from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
import { MessagingMessageParticipantService } from 'src/modules/messaging/message-participant-manager/services/messaging-message-participant.service';
import { isGroupEmail } from 'src/utils/is-group-email';
import { isWorkEmail } from 'src/utils/is-work-email';
@Injectable()
export class MessagingSaveMessagesAndEnqueueContactCreationService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
@InjectMessageQueue(MessageQueue.contactCreationQueue)
private readonly messageQueueService: MessageQueueService,
private readonly messageService: MessagingMessageService,
private readonly messageParticipantService: MessagingMessageParticipantService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
private readonly eventEmitter: EventEmitter2,
) {}
async saveMessagesAndEnqueueContactCreationJob(
messagesToSave: GmailMessage[],
messageChannel: MessageChannelWorkspaceEntity,
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
) {
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
const emailAliases = connectedAccount.emailAliases?.split(',') || [];
let savedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
const participantsWithMessageId = await workspaceDataSource?.transaction(
async (transactionManager: EntityManager) => {
const messageExternalIdsAndIdsMap =
await this.messageService.saveMessagesWithinTransaction(
messagesToSave,
connectedAccount,
messageChannel.id,
workspaceId,
transactionManager,
);
const participantsWithMessageId: (ParticipantWithMessageId & {
shouldCreateContact: boolean;
})[] = messagesToSave.flatMap((message) => {
const messageId = messageExternalIdsAndIdsMap.get(message.externalId);
return messageId
? message.participants.map((participant: Participant) => {
const fromHandle =
message.participants.find((p) => p.role === 'from')?.handle ||
'';
const isMessageSentByConnectedAccount =
emailAliases.includes(fromHandle);
const isParticipantConnectedAccount = emailAliases.includes(
participant.handle,
);
const isExcludedByNonProfessionalEmails =
messageChannel.excludeNonProfessionalEmails &&
!isWorkEmail(participant.handle);
const isExcludedByGroupEmails =
messageChannel.excludeGroupEmails &&
isGroupEmail(participant.handle);
const shouldCreateContact =
!isParticipantConnectedAccount &&
!isExcludedByNonProfessionalEmails &&
!isExcludedByGroupEmails &&
(messageChannel.contactAutoCreationPolicy ===
MessageChannelContactAutoCreationPolicy.SENT_AND_RECEIVED ||
(messageChannel.contactAutoCreationPolicy ===
MessageChannelContactAutoCreationPolicy.SENT &&
isMessageSentByConnectedAccount));
return {
...participant,
messageId,
shouldCreateContact,
};
})
: [];
});
savedMessageParticipants =
await this.messageParticipantService.saveMessageParticipants(
participantsWithMessageId,
workspaceId,
transactionManager,
);
return participantsWithMessageId;
},
);
this.eventEmitter.emit(`messageParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
messageParticipants: savedMessageParticipants,
});
if (messageChannel.isContactAutoCreationEnabled) {
const contactsToCreate = participantsWithMessageId.filter(
(participant) => participant.shouldCreateContact,
);
await this.messageQueueService.add<CreateCompanyAndContactJobData>(
CreateCompanyAndContactJob.name,
{
workspaceId,
connectedAccount,
contactsToCreate,
},
);
}
}
}

25
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/is-throttled.ts
View File

@ -1,25 +0,0 @@
import { MESSAGING_THROTTLE_DURATION } from 'src/modules/messaging/common/constants/messaging-throttle-duration';
export const isThrottled = (
syncStageStartedAt: string | null,
throttleFailureCount: number,
): boolean => {
if (!syncStageStartedAt) {
return false;
}
return (
computeThrottlePauseUntil(syncStageStartedAt, throttleFailureCount) >
new Date()
);
};
const computeThrottlePauseUntil = (
syncStageStartedAt: string,
throttleFailureCount: number,
): Date => {
return new Date(
new Date(syncStageStartedAt).getTime() +
MESSAGING_THROTTLE_DURATION * Math.pow(2, throttleFailureCount - 1),
);
};

2
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts
View File

@ -14,7 +14,7 @@ import {
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingGmailFullMessageListFetchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-full-message-list-fetch.service';
import { MessagingGmailPartialMessageListFetchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-partial-message-list-fetch.service';
import { isThrottled } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/is-throttled';
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
export type MessagingMessageListFetchJobData = {
messageChannelId: string;

2
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts
View File

@ -13,7 +13,7 @@ import {
MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingGmailMessagesImportService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service';
import { isThrottled } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/is-throttled';
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
export type MessagingMessagesImportJobData = {
messageChannelId: string;

2
packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts
View File

@ -21,9 +21,11 @@ import { MessagingMessagesImportJob } from 'src/modules/messaging/message-import
import { MessagingOngoingStaleJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job';
import { MessagingMessageImportManagerMessageChannelListener } from 'src/modules/messaging/message-import-manager/listeners/messaging-import-manager-message-channel.listener';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
WorkspaceDataSourceModule,
MessagingGmailDriverModule,
MessagingCommonModule,
TypeOrmModule.forFeature([Workspace], 'core'),

2
packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts
View File

@ -1,4 +1,4 @@
import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util';
import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant-manager/utils/is-email-blocklisted.util';
import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
// Todo: refactor this into several utils