From 3caf860848cab227d92db6b26209b76e692231fd Mon Sep 17 00:00:00 2001 From: bosiraphael <71827178+bosiraphael@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:23:31 +0100 Subject: [PATCH] 4285 timebox create google calendar full sync (#4442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * calendar module * wip * creating a folder for common files between calendar and messages * wip * wip * wip * wip * update calendar search filter * wip * working on full sync service * reorganizing folders * adding repositories * fix typo * working on full-sync service * Add calendarQueue to MessageQueue enum and update dependencies * start transaction * wip * add save and update functions for event * wip * save events * improving step by step * add calendar scope * fix nest modules imports * renaming * create calendar channel * create job for google calendar full-sync * call GoogleCalendarFullSyncJob after connected account creation * ask for scope conditionnally * fixes * create channels conditionnally * fix * fixes * fix FK bug * filter out canceled events * create save and update functions for calendarEventAttendee repository * saving messageParticipants is working * save calendarEventAttendees is working * add calendarEvent cleaner * calendar event cleaner is working * working on updating attendees * wip * reintroducing google-gmail endpoint to ensure smooth deploy * modify callbackURL * modify front url * changes to be able to merge * put back feature flag * fixes after PR comments * add feature flag check * remove unused modules * separate delete connected account associated job data in two jobs * fix error * rename calendar_v3 as calendarV3 * Update packages/twenty-server/src/workspace/calendar-and-messaging/utils/valueStringForBatchRawQuery.util.ts Co-authored-by: Jérémy M * improve readability * renaming to remove plural * renaming to remove plural * don't throw if no connected account is found * use calendar queue * modify usage of HttpService in fetch-by-batch * modify valuesStringForBatchRawQuery to improve api and return flattened values * fix auth module feature flag import * fix getFlattenedValuesAndValuesStringForBatchRawQuery --------- Co-authored-by: Jérémy M --- .../docs/start/self-hosting/self-hosting.mdx | 2 + packages/twenty-server/.env.example | 2 + packages/twenty-server/src/command.module.ts | 2 + .../src/core/auth/auth.module.ts | 14 +- .../google-apis-auth.controller.ts | 63 ++++ .../google-gmail-auth.controller.ts | 16 +- ...th.guard.ts => google-apis-oauth.guard.ts} | 2 +- .../google-apis-provider-enabled.guard.ts | 24 ++ .../google-gmail-provider-enabled.guard.ts | 21 -- ...mail.service.ts => google-apis.service.ts} | 82 +++++- ...rategy.ts => google-apis.auth.strategy.ts} | 30 +- .../environment/environment.service.ts | 11 + .../integrations/message-queue/jobs.module.ts | 24 +- .../message-queue/message-queue.constants.ts | 1 + .../create-company-and-contact.module.ts | 21 ++ .../create-company-and-contact.service.ts} | 10 +- .../create-company/create-company.module.ts | 2 +- .../create-company/create-company.service.ts | 0 .../create-contact/create-contact.module.ts | 4 +- .../create-contact/create-contact.service.ts | 2 +- .../blocklist/blocklist.module.ts | 2 +- .../blocklist/blocklist.service.ts | 0 .../connected-account.module.ts | 2 +- .../connected-account.service.ts | 0 ...ogle-apis-refresh-access-token.service.ts} | 4 +- .../types/batch-queries.ts | 5 + ...uesAndValuesStringForBatchRawQuery.util.ts | 56 ++++ .../src/workspace/calendar/calendar.module.ts | 41 +++ .../google-calendar-full-sync.command.ts | 66 +++++ ...workspace-calendar-sync-commands.module.ts | 19 ++ .../jobs/google-calendar-full-sync.job.ts | 53 ++++ ...alendar-channel-event-assocation.module.ts | 11 + ...endar-channel-event-association.service.ts | 172 +++++++++++ .../calendar-channel.module.ts | 11 + .../calendar-channel.service.ts | 76 +++++ .../calendar-event-attendee.module.ts | 11 + .../calendar-event-attendee.service.ts | 151 ++++++++++ .../calendar-event/calendar-event.module.ts | 11 + .../calendar-event/calendar-event.service.ts | 202 +++++++++++++ .../calendar-event-cleaner.module.ts | 13 + .../calendar-event-cleaner.service.ts | 20 ++ .../google-calendar-full-sync.service.ts | 274 ++++++++++++++++++ .../providers/calendar-providers.module.ts | 11 + .../google-calendar.provider.ts | 42 +++ .../calendar/types/calendar-event.ts | 31 ++ .../format-google-calendar-event.util.ts | 52 ++++ .../google-calendar-search-filter.util.ts | 9 + .../fetch-all-workspaces-messages.job.ts | 2 +- ...etch-workspace-messages-commands.module.ts | 2 +- .../commands/gmail-full-sync.command.ts | 2 +- .../commands/gmail-partial-sync.command.ts | 2 +- ...e-companies-and-contacts-after-sync.job.ts | 4 +- ...ed-account-associated-calendar-data.job.ts | 40 +++ ...-account-associated-messaging-data.job.ts} | 15 +- .../messaging/jobs/gmail-full-sync.job.ts | 6 +- .../messaging/jobs/gmail-partial-sync.job.ts | 6 +- .../messaging-connected-account.listener.ts | 24 +- .../workspace/messaging/messaging.module.ts | 27 +- .../message-find-many.pre-query.hook.ts | 4 +- .../messaging-query-hook.module.ts | 4 +- .../message-participant.module.ts | 2 +- .../message-participant.service.ts | 2 +- .../repositories/message/message.module.ts | 2 +- .../create-companies-and-contacts.module.ts | 21 -- .../services/fetch-by-batch.service.ts | 128 ++++++++ .../fetch-messages-by-batches.service.ts | 130 +-------- .../services/gmail-full-sync.service.ts | 6 +- .../services/gmail-partial-sync.service.ts | 4 +- ...ve-messages-and-create-contacts.service.ts | 4 +- ...-filter.ts => gmail-search-filter.util.ts} | 0 .../repositories/person/person.module.ts | 3 +- .../repositories/person/person.service.ts | 0 .../workspace-member.module.ts | 3 +- .../workspace-member.service.ts | 0 .../calendar-event.object-metadata.ts | 13 +- .../src/workspace/workspace.module.ts | 2 + 76 files changed, 1856 insertions(+), 280 deletions(-) create mode 100644 packages/twenty-server/src/core/auth/controllers/google-apis-auth.controller.ts rename packages/twenty-server/src/core/auth/guards/{google-gmail-oauth.guard.ts => google-apis-oauth.guard.ts} (90%) create mode 100644 packages/twenty-server/src/core/auth/guards/google-apis-provider-enabled.guard.ts delete mode 100644 packages/twenty-server/src/core/auth/guards/google-gmail-provider-enabled.guard.ts rename packages/twenty-server/src/core/auth/services/{google-gmail.service.ts => google-apis.service.ts} (50%) rename packages/twenty-server/src/core/auth/strategies/{google-gmail.auth.strategy.ts => google-apis.auth.strategy.ts} (69%) create mode 100644 packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module.ts rename packages/twenty-server/src/workspace/{messaging/services/create-companies-and-contacts/create-companies-and-contacts.service.ts => auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service.ts} (87%) rename packages/twenty-server/src/workspace/{messaging/services => auto-companies-and-contacts-creation}/create-company/create-company.module.ts (76%) rename packages/twenty-server/src/workspace/{messaging/services => auto-companies-and-contacts-creation}/create-company/create-company.service.ts (100%) rename packages/twenty-server/src/workspace/{messaging/services => auto-companies-and-contacts-creation}/create-contact/create-contact.module.ts (60%) rename packages/twenty-server/src/workspace/{messaging/services => auto-companies-and-contacts-creation}/create-contact/create-contact.service.ts (94%) rename packages/twenty-server/src/workspace/{messaging => calendar-and-messaging}/repositories/blocklist/blocklist.module.ts (72%) rename packages/twenty-server/src/workspace/{messaging => calendar-and-messaging}/repositories/blocklist/blocklist.service.ts (100%) rename packages/twenty-server/src/workspace/{messaging => calendar-and-messaging}/repositories/connected-account/connected-account.module.ts (70%) rename packages/twenty-server/src/workspace/{messaging => calendar-and-messaging}/repositories/connected-account/connected-account.service.ts (100%) rename packages/twenty-server/src/workspace/{messaging/services/gmail-refresh-access-token.service.ts => calendar-and-messaging/services/google-apis-refresh-access-token.service.ts} (90%) create mode 100644 packages/twenty-server/src/workspace/calendar-and-messaging/types/batch-queries.ts create mode 100644 packages/twenty-server/src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util.ts create mode 100644 packages/twenty-server/src/workspace/calendar/calendar.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/commands/google-calendar-full-sync.command.ts create mode 100644 packages/twenty-server/src/workspace/calendar/commands/workspace-calendar-sync-commands.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/jobs/google-calendar-full-sync.job.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.service.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.service.ts create mode 100644 packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service.ts create mode 100644 packages/twenty-server/src/workspace/calendar/services/google-calendar-full-sync.service.ts create mode 100644 packages/twenty-server/src/workspace/calendar/services/providers/calendar-providers.module.ts create mode 100644 packages/twenty-server/src/workspace/calendar/services/providers/google-calendar/google-calendar.provider.ts create mode 100644 packages/twenty-server/src/workspace/calendar/types/calendar-event.ts create mode 100644 packages/twenty-server/src/workspace/calendar/utils/format-google-calendar-event.util.ts create mode 100644 packages/twenty-server/src/workspace/calendar/utils/google-calendar-search-filter.util.ts create mode 100644 packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job.ts rename packages/twenty-server/src/workspace/messaging/jobs/{delete-connected-acount-associated-data.job.ts => delete-connected-account-associated-messaging-data.job.ts} (63%) delete mode 100644 packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module.ts create mode 100644 packages/twenty-server/src/workspace/messaging/services/fetch-by-batch.service.ts rename packages/twenty-server/src/workspace/messaging/utils/{gmail-search-filter.ts => gmail-search-filter.util.ts} (100%) rename packages/twenty-server/src/workspace/{messaging => }/repositories/person/person.module.ts (67%) rename packages/twenty-server/src/workspace/{messaging => }/repositories/person/person.service.ts (100%) rename packages/twenty-server/src/workspace/{messaging => }/repositories/workspace-member/workspace-member.module.ts (65%) rename packages/twenty-server/src/workspace/{messaging => }/repositories/workspace-member/workspace-member.service.ts (100%) diff --git a/packages/twenty-docs/docs/start/self-hosting/self-hosting.mdx b/packages/twenty-docs/docs/start/self-hosting/self-hosting.mdx index b138bfda3..343c7aca0 100644 --- a/packages/twenty-docs/docs/start/self-hosting/self-hosting.mdx +++ b/packages/twenty-docs/docs/start/self-hosting/self-hosting.mdx @@ -68,7 +68,9 @@ import TabItem from '@theme/TabItem'; | Observable { + if ( + !this.environmentService.isMessagingProviderGmailEnabled() && + !this.environmentService.isCalendarProviderGoogleEnabled() + ) { + throw new NotFoundException('Google apis auth is not enabled'); + } + + new GoogleAPIsStrategy(this.environmentService); + + return true; + } +} diff --git a/packages/twenty-server/src/core/auth/guards/google-gmail-provider-enabled.guard.ts b/packages/twenty-server/src/core/auth/guards/google-gmail-provider-enabled.guard.ts deleted file mode 100644 index d5ea0e292..000000000 --- a/packages/twenty-server/src/core/auth/guards/google-gmail-provider-enabled.guard.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Injectable, CanActivate, NotFoundException } from '@nestjs/common'; - -import { Observable } from 'rxjs'; - -import { GoogleGmailStrategy } from 'src/core/auth/strategies/google-gmail.auth.strategy'; -import { EnvironmentService } from 'src/integrations/environment/environment.service'; - -@Injectable() -export class GoogleGmailProviderEnabledGuard implements CanActivate { - constructor(private readonly environmentService: EnvironmentService) {} - - canActivate(): boolean | Promise | Observable { - if (!this.environmentService.isMessagingProviderGmailEnabled()) { - throw new NotFoundException('Gmail auth is not enabled'); - } - - new GoogleGmailStrategy(this.environmentService); - - return true; - } -} diff --git a/packages/twenty-server/src/core/auth/services/google-gmail.service.ts b/packages/twenty-server/src/core/auth/services/google-apis.service.ts similarity index 50% rename from packages/twenty-server/src/core/auth/services/google-gmail.service.ts rename to packages/twenty-server/src/core/auth/services/google-apis.service.ts index 8af8605ff..1e1fffd3b 100644 --- a/packages/twenty-server/src/core/auth/services/google-gmail.service.ts +++ b/packages/twenty-server/src/core/auth/services/google-apis.service.ts @@ -1,6 +1,8 @@ import { ConflictException, Inject, Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; import { v4 } from 'uuid'; +import { Repository } from 'typeorm'; import { DataSourceService } from 'src/metadata/data-source/data-source.service'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; @@ -11,14 +13,28 @@ import { GmailFullSyncJob, GmailFullSyncJobData, } from 'src/workspace/messaging/jobs/gmail-full-sync.job'; +import { + GoogleCalendarFullSyncJob, + GoogleCalendarFullSyncJobData, +} from 'src/workspace/calendar/jobs/google-calendar-full-sync.job'; +import { EnvironmentService } from 'src/integrations/environment/environment.service'; +import { + FeatureFlagEntity, + FeatureFlagKeys, +} from 'src/core/feature-flag/feature-flag.entity'; @Injectable() -export class GoogleGmailService { +export class GoogleAPIsService { constructor( private readonly dataSourceService: DataSourceService, private readonly typeORMService: TypeORMService, @Inject(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, + @Inject(MessageQueue.calendarQueue) + private readonly calendarQueueService: MessageQueueService, + private readonly environmentService: EnvironmentService, + @InjectRepository(FeatureFlagEntity, 'core') + private readonly featureFlagRepository: Repository, ) {} providerName = 'google'; @@ -53,6 +69,12 @@ export class GoogleGmailService { const connectedAccountId = v4(); + const IsCalendarEnabled = await this.featureFlagRepository.findOneBy({ + workspaceId, + key: FeatureFlagKeys.IsCalendarEnabled, + value: true, + }); + await workspaceDataSource?.transaction(async (manager) => { await manager.query( `INSERT INTO ${dataSourceMetadata.schema}."connectedAccount" ("id", "handle", "provider", "accessToken", "refreshToken", "accountOwnerId") VALUES ($1, $2, $3, $4, $5, $6)`, @@ -66,22 +88,52 @@ export class GoogleGmailService { ], ); - await manager.query( - `INSERT INTO ${dataSourceMetadata.schema}."messageChannel" ("visibility", "handle", "connectedAccountId", "type") VALUES ($1, $2, $3, $4)`, - ['share_everything', handle, connectedAccountId, 'email'], - ); + if (this.environmentService.isMessagingProviderGmailEnabled()) { + await manager.query( + `INSERT INTO ${dataSourceMetadata.schema}."messageChannel" ("visibility", "handle", "connectedAccountId", "type") VALUES ($1, $2, $3, $4)`, + ['share_everything', handle, connectedAccountId, 'email'], + ); + } + + if ( + this.environmentService.isCalendarProviderGoogleEnabled() && + IsCalendarEnabled + ) { + await manager.query( + `INSERT INTO ${dataSourceMetadata.schema}."calendarChannel" ("visibility", "handle", "connectedAccountId") VALUES ($1, $2, $3)`, + ['SHARE_EVERYTHING', handle, connectedAccountId], + ); + } }); - await this.messageQueueService.add( - GmailFullSyncJob.name, - { - workspaceId, - connectedAccountId, - }, - { - retryLimit: 2, - }, - ); + if (this.environmentService.isMessagingProviderGmailEnabled()) { + await this.messageQueueService.add( + GmailFullSyncJob.name, + { + workspaceId, + connectedAccountId, + }, + { + retryLimit: 2, + }, + ); + } + + if ( + this.environmentService.isCalendarProviderGoogleEnabled() && + IsCalendarEnabled + ) { + await this.calendarQueueService.add( + GoogleCalendarFullSyncJob.name, + { + workspaceId, + connectedAccountId, + }, + { + retryLimit: 2, + }, + ); + } return; } diff --git a/packages/twenty-server/src/core/auth/strategies/google-gmail.auth.strategy.ts b/packages/twenty-server/src/core/auth/strategies/google-apis.auth.strategy.ts similarity index 69% rename from packages/twenty-server/src/core/auth/strategies/google-gmail.auth.strategy.ts rename to packages/twenty-server/src/core/auth/strategies/google-apis.auth.strategy.ts index 09e854533..91a9f624f 100644 --- a/packages/twenty-server/src/core/auth/strategies/google-gmail.auth.strategy.ts +++ b/packages/twenty-server/src/core/auth/strategies/google-apis.auth.strategy.ts @@ -6,7 +6,7 @@ import { Request } from 'express'; import { EnvironmentService } from 'src/integrations/environment/environment.service'; -export type GoogleGmailRequest = Request & { +export type GoogleAPIsRequest = Request & { user: { firstName?: string | null; lastName?: string | null; @@ -20,20 +20,28 @@ export type GoogleGmailRequest = Request & { }; @Injectable() -export class GoogleGmailStrategy extends PassportStrategy( +export class GoogleAPIsStrategy extends PassportStrategy( Strategy, - 'google-gmail', + 'google-apis', ) { constructor(environmentService: EnvironmentService) { + const scope = ['email', 'profile']; + + if (environmentService.isMessagingProviderGmailEnabled()) { + scope.push('https://www.googleapis.com/auth/gmail.readonly'); + } + + if (environmentService.isCalendarProviderGoogleEnabled()) { + scope.push('https://www.googleapis.com/auth/calendar'); + } + super({ clientID: environmentService.getAuthGoogleClientId(), clientSecret: environmentService.getAuthGoogleClientSecret(), - callbackURL: environmentService.getMessagingProviderGmailCallbackUrl(), - scope: [ - 'email', - 'profile', - 'https://www.googleapis.com/auth/gmail.readonly', - ], + callbackURL: environmentService.isCalendarProviderGoogleEnabled() + ? environmentService.getAuthGoogleAPIsCallbackUrl() + : environmentService.getMessagingProviderGmailCallbackUrl(), + scope, passReqToCallback: true, }); } @@ -52,7 +60,7 @@ export class GoogleGmailStrategy extends PassportStrategy( } async validate( - request: GoogleGmailRequest, + request: GoogleAPIsRequest, accessToken: string, refreshToken: string, profile: any, @@ -65,7 +73,7 @@ export class GoogleGmailStrategy extends PassportStrategy( ? JSON.parse(request.query.state) : undefined; - const user: GoogleGmailRequest['user'] = { + const user: GoogleAPIsRequest['user'] = { email: emails[0].value, firstName: name.givenName, lastName: name.familyName, diff --git a/packages/twenty-server/src/integrations/environment/environment.service.ts b/packages/twenty-server/src/integrations/environment/environment.service.ts index 6b35c9bda..05ee2184e 100644 --- a/packages/twenty-server/src/integrations/environment/environment.service.ts +++ b/packages/twenty-server/src/integrations/environment/environment.service.ts @@ -155,12 +155,23 @@ export class EnvironmentService { ); } + isCalendarProviderGoogleEnabled(): boolean { + return ( + this.configService.get('CALENDAR_PROVIDER_GOOGLE_ENABLED') ?? + false + ); + } + getMessagingProviderGmailCallbackUrl(): string | undefined { return this.configService.get( 'MESSAGING_PROVIDER_GMAIL_CALLBACK_URL', ); } + getAuthGoogleAPIsCallbackUrl(): string | undefined { + return this.configService.get('AUTH_GOOGLE_APIS_CALLBACK_URL'); + } + isAuthGoogleEnabled(): boolean { return this.configService.get('AUTH_GOOGLE_ENABLED') ?? false; } diff --git a/packages/twenty-server/src/integrations/message-queue/jobs.module.ts b/packages/twenty-server/src/integrations/message-queue/jobs.module.ts index 1d97d71e3..3be853f8e 100644 --- a/packages/twenty-server/src/integrations/message-queue/jobs.module.ts +++ b/packages/twenty-server/src/integrations/message-queue/jobs.module.ts @@ -17,15 +17,15 @@ import { EmailSenderJob } from 'src/integrations/email/email-sender.job'; import { UserModule } from 'src/core/user/user.module'; import { EnvironmentModule } from 'src/integrations/environment/environment.module'; import { FetchAllWorkspacesMessagesJob } from 'src/workspace/messaging/commands/crons/fetch-all-workspaces-messages.job'; -import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module'; +import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module'; import { MatchMessageParticipantJob } from 'src/workspace/messaging/jobs/match-message-participant.job'; import { CreateCompaniesAndContactsAfterSyncJob } from 'src/workspace/messaging/jobs/create-companies-and-contacts-after-sync.job'; -import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module'; +import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module'; import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module'; import { MessageParticipantModule } from 'src/workspace/messaging/repositories/message-participant/message-participant.module'; import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module'; import { DataSeedDemoWorkspaceJob } from 'src/database/commands/data-seed-demo-workspace/jobs/data-seed-demo-workspace.job'; -import { DeleteConnectedAccountAssociatedDataJob } from 'src/workspace/messaging/jobs/delete-connected-acount-associated-data.job'; +import { DeleteConnectedAccountAssociatedMessagingDataJob } from 'src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job'; import { ThreadCleanerModule } from 'src/workspace/messaging/services/thread-cleaner/thread-cleaner.module'; import { UpdateSubscriptionJob } from 'src/core/billing/jobs/update-subscription.job'; import { BillingModule } from 'src/core/billing/billing.module'; @@ -33,9 +33,13 @@ import { UserWorkspaceModule } from 'src/core/user-workspace/user-workspace.modu import { StripeModule } from 'src/core/billing/stripe/stripe.module'; import { Workspace } from 'src/core/workspace/workspace.entity'; import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity'; +import { CalendarModule } from 'src/workspace/calendar/calendar.module'; import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity'; +import { GoogleCalendarFullSyncJob } from 'src/workspace/calendar/jobs/google-calendar-full-sync.job'; +import { CalendarEventCleanerModule } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module'; import { RecordPositionBackfillJob } from 'src/workspace/workspace-query-runner/jobs/record-position-backfill.job'; import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runner/services/record-position-backfill-module'; +import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job'; @Module({ imports: [ @@ -49,9 +53,11 @@ import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runn MessagingModule, MessageParticipantModule, MessageChannelModule, + CalendarModule, ObjectMetadataModule, StripeModule, ThreadCleanerModule, + CalendarEventCleanerModule, TypeORMModule, TypeOrmModule.forFeature([Workspace, FeatureFlagEntity], 'core'), TypeOrmModule.forFeature([DataSourceEntity], 'metadata'), @@ -69,6 +75,10 @@ import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runn provide: GmailPartialSyncJob.name, useClass: GmailPartialSyncJob, }, + { + provide: GoogleCalendarFullSyncJob.name, + useClass: GoogleCalendarFullSyncJob, + }, { provide: CallWebhookJobsJob.name, useClass: CallWebhookJobsJob, @@ -99,8 +109,12 @@ import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runn useClass: DataSeedDemoWorkspaceJob, }, { - provide: DeleteConnectedAccountAssociatedDataJob.name, - useClass: DeleteConnectedAccountAssociatedDataJob, + provide: DeleteConnectedAccountAssociatedMessagingDataJob.name, + useClass: DeleteConnectedAccountAssociatedMessagingDataJob, + }, + { + provide: DeleteConnectedAccountAssociatedCalendarDataJob.name, + useClass: DeleteConnectedAccountAssociatedCalendarDataJob, }, { provide: UpdateSubscriptionJob.name, useClass: UpdateSubscriptionJob }, { diff --git a/packages/twenty-server/src/integrations/message-queue/message-queue.constants.ts b/packages/twenty-server/src/integrations/message-queue/message-queue.constants.ts index 25da11238..7cb1fae15 100644 --- a/packages/twenty-server/src/integrations/message-queue/message-queue.constants.ts +++ b/packages/twenty-server/src/integrations/message-queue/message-queue.constants.ts @@ -6,6 +6,7 @@ export enum MessageQueue { webhookQueue = 'webhook-queue', cronQueue = 'cron-queue', emailQueue = 'email-queue', + calendarQueue = 'calendar-queue', billingQueue = 'billing-queue', recordPositionBackfillQueue = 'record-position-backfill-queue', } diff --git a/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module.ts b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module.ts new file mode 100644 index 000000000..b46d37253 --- /dev/null +++ b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; + +import { PersonModule } from 'src/workspace/repositories/person/person.module'; +import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module'; +import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service'; +import { CreateCompanyModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.module'; +import { CreateContactModule } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.module'; +import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [ + WorkspaceDataSourceModule, + CreateContactModule, + CreateCompanyModule, + WorkspaceMemberModule, + PersonModule, + ], + providers: [CreateCompanyAndContactService], + exports: [CreateCompanyAndContactService], +}) +export class CreateCompaniesAndContactsModule {} diff --git a/packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service.ts b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service.ts similarity index 87% rename from packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service.ts rename to packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service.ts index 8492261f0..c9cc641ee 100644 --- a/packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service.ts +++ b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service.ts @@ -5,16 +5,16 @@ import compact from 'lodash/compact'; import { Participant } from 'src/workspace/messaging/types/gmail-message'; import { getDomainNameFromHandle } from 'src/workspace/messaging/utils/get-domain-name-from-handle.util'; -import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service'; -import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service'; -import { PersonService } from 'src/workspace/messaging/repositories/person/person.service'; -import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service'; +import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service'; +import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service'; +import { PersonService } from 'src/workspace/repositories/person/person.service'; +import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service'; import { getUniqueParticipantsAndHandles } from 'src/workspace/messaging/utils/get-unique-participants-and-handles.util'; import { filterOutParticipantsFromCompanyOrWorkspace } from 'src/workspace/messaging/utils/filter-out-participants-from-company-or-workspace.util'; import { isWorkEmail } from 'src/utils/is-work-email'; @Injectable() -export class CreateCompaniesAndContactsService { +export class CreateCompanyAndContactService { constructor( private readonly personService: PersonService, private readonly createContactService: CreateContactService, diff --git a/packages/twenty-server/src/workspace/messaging/services/create-company/create-company.module.ts b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company/create-company.module.ts similarity index 76% rename from packages/twenty-server/src/workspace/messaging/services/create-company/create-company.module.ts rename to packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company/create-company.module.ts index b9ce318f7..663f007a3 100644 --- a/packages/twenty-server/src/workspace/messaging/services/create-company/create-company.module.ts +++ b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company/create-company.module.ts @@ -1,7 +1,7 @@ import { Module } from '@nestjs/common'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; -import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service'; +import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service'; import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module'; @Module({ diff --git a/packages/twenty-server/src/workspace/messaging/services/create-company/create-company.service.ts b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service.ts similarity index 100% rename from packages/twenty-server/src/workspace/messaging/services/create-company/create-company.service.ts rename to packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service.ts diff --git a/packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.module.ts b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.module.ts similarity index 60% rename from packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.module.ts rename to packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.module.ts index 6a20b9fe6..20d556744 100644 --- a/packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.module.ts +++ b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.module.ts @@ -1,8 +1,8 @@ import { Module } from '@nestjs/common'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; -import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service'; -import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module'; +import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service'; +import { PersonModule } from 'src/workspace/repositories/person/person.module'; @Module({ imports: [WorkspaceDataSourceModule, PersonModule], diff --git a/packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.service.ts b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service.ts similarity index 94% rename from packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.service.ts rename to packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service.ts index 2e23af46b..81966d104 100644 --- a/packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.service.ts +++ b/packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service.ts @@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common'; import { EntityManager } from 'typeorm'; import { v4 } from 'uuid'; -import { PersonService } from 'src/workspace/messaging/repositories/person/person.service'; +import { PersonService } from 'src/workspace/repositories/person/person.service'; import { getFirstNameAndLastNameFromHandleAndDisplayName } from 'src/workspace/messaging/utils/get-first-name-and-last-name-from-handle-and-display-name.util'; type ContactToCreate = { diff --git a/packages/twenty-server/src/workspace/messaging/repositories/blocklist/blocklist.module.ts b/packages/twenty-server/src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module.ts similarity index 72% rename from packages/twenty-server/src/workspace/messaging/repositories/blocklist/blocklist.module.ts rename to packages/twenty-server/src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module.ts index 17f434665..aa570ea61 100644 --- a/packages/twenty-server/src/workspace/messaging/repositories/blocklist/blocklist.module.ts +++ b/packages/twenty-server/src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module.ts @@ -1,6 +1,6 @@ import { Module } from '@nestjs/common'; -import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service'; +import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; @Module({ diff --git a/packages/twenty-server/src/workspace/messaging/repositories/blocklist/blocklist.service.ts b/packages/twenty-server/src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service.ts similarity index 100% rename from packages/twenty-server/src/workspace/messaging/repositories/blocklist/blocklist.service.ts rename to packages/twenty-server/src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service.ts diff --git a/packages/twenty-server/src/workspace/messaging/repositories/connected-account/connected-account.module.ts b/packages/twenty-server/src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module.ts similarity index 70% rename from packages/twenty-server/src/workspace/messaging/repositories/connected-account/connected-account.module.ts rename to packages/twenty-server/src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module.ts index 12292e445..f2b36f0e1 100644 --- a/packages/twenty-server/src/workspace/messaging/repositories/connected-account/connected-account.module.ts +++ b/packages/twenty-server/src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module.ts @@ -1,6 +1,6 @@ import { Module } from '@nestjs/common'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; @Module({ diff --git a/packages/twenty-server/src/workspace/messaging/repositories/connected-account/connected-account.service.ts b/packages/twenty-server/src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service.ts similarity index 100% rename from packages/twenty-server/src/workspace/messaging/repositories/connected-account/connected-account.service.ts rename to packages/twenty-server/src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service.ts diff --git a/packages/twenty-server/src/workspace/messaging/services/gmail-refresh-access-token.service.ts b/packages/twenty-server/src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service.ts similarity index 90% rename from packages/twenty-server/src/workspace/messaging/services/gmail-refresh-access-token.service.ts rename to packages/twenty-server/src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service.ts index a94a372ef..b30c1ded7 100644 --- a/packages/twenty-server/src/workspace/messaging/services/gmail-refresh-access-token.service.ts +++ b/packages/twenty-server/src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service.ts @@ -3,10 +3,10 @@ import { Injectable } from '@nestjs/common'; import axios from 'axios'; import { EnvironmentService } from 'src/integrations/environment/environment.service'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; @Injectable() -export class GmailRefreshAccessTokenService { +export class GoogleAPIsRefreshAccessTokenService { constructor( private readonly environmentService: EnvironmentService, private readonly connectedAccountService: ConnectedAccountService, diff --git a/packages/twenty-server/src/workspace/calendar-and-messaging/types/batch-queries.ts b/packages/twenty-server/src/workspace/calendar-and-messaging/types/batch-queries.ts new file mode 100644 index 000000000..c072a6e71 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar-and-messaging/types/batch-queries.ts @@ -0,0 +1,5 @@ +type Query = { + uri: string; +}; + +export type BatchQueries = Query[]; diff --git a/packages/twenty-server/src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util.ts b/packages/twenty-server/src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util.ts new file mode 100644 index 000000000..73c7c6eb1 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util.ts @@ -0,0 +1,56 @@ +export const valuesStringForBatchRawQuery = ( + values: { + [key: string]: any; + }[], + typesArray: string[] = [], +) => { + const castedValues = values.reduce((acc, _, rowIndex) => { + const numberOfColumns = typesArray.length; + + const rowValues = Array.from( + { length: numberOfColumns }, + (_, columnIndex) => { + const placeholder = `$${rowIndex * numberOfColumns + columnIndex + 1}`; + const typeCast = typesArray[columnIndex] + ? `::${typesArray[columnIndex]}` + : ''; + + return `${placeholder}${typeCast}`; + }, + ).join(', '); + + acc.push(`(${rowValues})`); + + return acc; + }, [] as string[]); + + return castedValues.join(', '); +}; + +export const getFlattenedValuesAndValuesStringForBatchRawQuery = ( + values: { + [key: string]: any; + }[], + keyTypeMap: { + [key: string]: string; + }, +): { + flattenedValues: any[]; + valuesString: string; +} => { + const keysToInsert = Object.keys(keyTypeMap); + + const flattenedValues = values.flatMap((value) => + keysToInsert.map((key) => value[key]), + ); + + const valuesString = valuesStringForBatchRawQuery( + values, + Object.values(keyTypeMap), + ); + + return { + flattenedValues, + valuesString, + }; +}; diff --git a/packages/twenty-server/src/workspace/calendar/calendar.module.ts b/packages/twenty-server/src/workspace/calendar/calendar.module.ts new file mode 100644 index 000000000..1a209e941 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/calendar.module.ts @@ -0,0 +1,41 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity'; +import { EnvironmentModule } from 'src/integrations/environment/environment.module'; +import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module'; +import { BlocklistModule } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module'; +import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module'; +import { CalendarChannelEventAssociationModule } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module'; +import { CalendarChannelModule } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.module'; +import { CalendarEventAttendeeModule } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module'; +import { CalendarEventModule } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.module'; +import { CalendarEventCleanerModule } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module'; +import { GoogleCalendarFullSyncService } from 'src/workspace/calendar/services/google-calendar-full-sync.service'; +import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider'; +import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module'; +import { PersonModule } from 'src/workspace/repositories/person/person.module'; +import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module'; +import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [ + EnvironmentModule, + WorkspaceDataSourceModule, + ConnectedAccountModule, + CalendarChannelModule, + CalendarChannelEventAssociationModule, + CalendarEventModule, + CalendarEventAttendeeModule, + CreateCompaniesAndContactsModule, + WorkspaceMemberModule, + TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), + CompanyModule, + PersonModule, + BlocklistModule, + CalendarEventCleanerModule, + ], + providers: [GoogleCalendarFullSyncService, GoogleCalendarClientProvider], + exports: [GoogleCalendarFullSyncService], +}) +export class CalendarModule {} diff --git a/packages/twenty-server/src/workspace/calendar/commands/google-calendar-full-sync.command.ts b/packages/twenty-server/src/workspace/calendar/commands/google-calendar-full-sync.command.ts new file mode 100644 index 000000000..139aa0255 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/commands/google-calendar-full-sync.command.ts @@ -0,0 +1,66 @@ +import { Inject } from '@nestjs/common'; + +import { Command, CommandRunner, Option } from 'nest-commander'; + +import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; +import { + GoogleCalendarFullSyncJobData, + GoogleCalendarFullSyncJob, +} from 'src/workspace/calendar/jobs/google-calendar-full-sync.job'; + +interface GoogleCalendarFullSyncOptions { + workspaceId: string; +} + +@Command({ + name: 'workspace:google-calendar-full-sync', + description: + 'Start google calendar full-sync for all workspaceMembers in a workspace.', +}) +export class GoogleCalendarFullSyncCommand extends CommandRunner { + constructor( + @Inject(MessageQueue.messagingQueue) + private readonly messageQueueService: MessageQueueService, + private readonly connectedAccountService: ConnectedAccountService, + ) { + super(); + } + + async run( + _passedParam: string[], + options: GoogleCalendarFullSyncOptions, + ): Promise { + await this.fetchWorkspaceCalendars(options.workspaceId); + + return; + } + + @Option({ + flags: '-w, --workspace-id [workspace_id]', + description: 'workspace id', + required: true, + }) + parseWorkspaceId(value: string): string { + return value; + } + + private async fetchWorkspaceCalendars(workspaceId: string): Promise { + const connectedAccounts = + await this.connectedAccountService.getAll(workspaceId); + + for (const connectedAccount of connectedAccounts) { + await this.messageQueueService.add( + GoogleCalendarFullSyncJob.name, + { + workspaceId, + connectedAccountId: connectedAccount.id, + }, + { + retryLimit: 2, + }, + ); + } + } +} diff --git a/packages/twenty-server/src/workspace/calendar/commands/workspace-calendar-sync-commands.module.ts b/packages/twenty-server/src/workspace/calendar/commands/workspace-calendar-sync-commands.module.ts new file mode 100644 index 000000000..4443be7dd --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/commands/workspace-calendar-sync-commands.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity'; +import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; +import { DataSourceModule } from 'src/metadata/data-source/data-source.module'; +import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module'; +import { GoogleCalendarFullSyncCommand } from 'src/workspace/calendar/commands/google-calendar-full-sync.command'; + +@Module({ + imports: [ + DataSourceModule, + TypeORMModule, + TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), + ConnectedAccountModule, + ], + providers: [GoogleCalendarFullSyncCommand], +}) +export class WorkspaceCalendarSyncCommandsModule {} diff --git a/packages/twenty-server/src/workspace/calendar/jobs/google-calendar-full-sync.job.ts b/packages/twenty-server/src/workspace/calendar/jobs/google-calendar-full-sync.job.ts new file mode 100644 index 000000000..36e17ca1f --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/jobs/google-calendar-full-sync.job.ts @@ -0,0 +1,53 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface'; + +import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service'; +import { GoogleCalendarFullSyncService } from 'src/workspace/calendar/services/google-calendar-full-sync.service'; + +export type GoogleCalendarFullSyncJobData = { + workspaceId: string; + connectedAccountId: string; + nextPageToken?: string; +}; + +@Injectable() +export class GoogleCalendarFullSyncJob + implements MessageQueueJob +{ + private readonly logger = new Logger(GoogleCalendarFullSyncJob.name); + + constructor( + private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService, + private readonly googleCalendarFullSyncService: GoogleCalendarFullSyncService, + ) {} + + async handle(data: GoogleCalendarFullSyncJobData): Promise { + this.logger.log( + `google calendar full-sync for workspace ${ + data.workspaceId + } and account ${data.connectedAccountId} ${ + data.nextPageToken ? `and ${data.nextPageToken} pageToken` : '' + }`, + ); + try { + await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken( + data.workspaceId, + data.connectedAccountId, + ); + } catch (e) { + this.logger.error( + `Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`, + e, + ); + + return; + } + + await this.googleCalendarFullSyncService.startGoogleCalendarFullSync( + data.workspaceId, + data.connectedAccountId, + data.nextPageToken, + ); + } +} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module.ts new file mode 100644 index 000000000..88b254027 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; + +import { CalendarChannelEventAssociationService } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service'; +import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [WorkspaceDataSourceModule], + providers: [CalendarChannelEventAssociationService], + exports: [CalendarChannelEventAssociationService], +}) +export class CalendarChannelEventAssociationModule {} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service.ts new file mode 100644 index 000000000..a4d52fc77 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service.ts @@ -0,0 +1,172 @@ +import { Injectable } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service'; +import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record'; +import { CalendarChannelEventAssociationObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel-event-association.object-metadata'; +import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util'; + +@Injectable() +export class CalendarChannelEventAssociationService { + constructor( + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) {} + + public async getByEventExternalIdsAndCalendarChannelId( + eventExternalIds: string[], + calendarChannelId: string, + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + if (eventExternalIds.length === 0) { + return []; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation" + WHERE "eventExternalId" = ANY($1) AND "calendarChannelId" = $2`, + [eventExternalIds, calendarChannelId], + workspaceId, + transactionManager, + ); + } + + public async deleteByEventExternalIdsAndCalendarChannelId( + eventExternalIds: string[], + calendarChannelId: string, + workspaceId: string, + transactionManager?: EntityManager, + ) { + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + await this.workspaceDataSourceService.executeRawQuery( + `DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "eventExternalId" = ANY($1) AND "calendarChannelId" = $2`, + [eventExternalIds, calendarChannelId], + workspaceId, + transactionManager, + ); + } + + public async getByCalendarChannelIds( + calendarChannelIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + if (calendarChannelIds.length === 0) { + return []; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation" + WHERE "calendarChannelId" = ANY($1)`, + [calendarChannelIds], + workspaceId, + transactionManager, + ); + } + + public async deleteByCalendarChannelIds( + calendarChannelIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ) { + if (calendarChannelIds.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + await this.workspaceDataSourceService.executeRawQuery( + `DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "calendarChannelId" = ANY($1)`, + [calendarChannelIds], + workspaceId, + transactionManager, + ); + } + + public async deleteByIds( + ids: string[], + workspaceId: string, + transactionManager?: EntityManager, + ) { + if (ids.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + await this.workspaceDataSourceService.executeRawQuery( + `DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "id" = ANY($1)`, + [ids], + workspaceId, + transactionManager, + ); + } + + public async getByCalendarEventIds( + calendarEventIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + if (calendarEventIds.length === 0) { + return []; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation" + WHERE "calendarEventId" = ANY($1)`, + [calendarEventIds], + workspaceId, + transactionManager, + ); + } + + public async saveCalendarChannelEventAssociations( + calendarChannelEventAssociations: Omit< + ObjectRecord, + 'id' | 'createdAt' | 'updatedAt' | 'calendarChannel' | 'calendarEvent' + >[], + workspaceId: string, + transactionManager?: EntityManager, + ) { + if (calendarChannelEventAssociations.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const { + flattenedValues: calendarChannelEventAssociationValues, + valuesString, + } = getFlattenedValuesAndValuesStringForBatchRawQuery( + calendarChannelEventAssociations, + { + calendarChannelId: 'uuid', + calendarEventId: 'uuid', + eventExternalId: 'text', + }, + ); + + await this.workspaceDataSourceService.executeRawQuery( + `INSERT INTO ${dataSourceSchema}."calendarChannelEventAssociation" ("calendarChannelId", "calendarEventId", "eventExternalId") + VALUES ${valuesString}`, + calendarChannelEventAssociationValues, + workspaceId, + transactionManager, + ); + } +} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.module.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.module.ts new file mode 100644 index 000000000..bfa02c465 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; + +import { CalendarChannelService } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.service'; +import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [WorkspaceDataSourceModule], + providers: [CalendarChannelService], + exports: [CalendarChannelService], +}) +export class CalendarChannelModule {} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.service.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.service.ts new file mode 100644 index 000000000..d97e9ef42 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.service.ts @@ -0,0 +1,76 @@ +import { Injectable } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service'; +import { CalendarChannelObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel.object-metadata'; +import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record'; + +@Injectable() +export class CalendarChannelService { + constructor( + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) {} + + public async getByConnectedAccountId( + connectedAccountId: string, + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarChannel" WHERE "connectedAccountId" = $1 LIMIT 1`, + [connectedAccountId], + workspaceId, + transactionManager, + ); + } + + public async getFirstByConnectedAccountIdOrFail( + connectedAccountId: string, + workspaceId: string, + ): Promise> { + const calendarChannels = await this.getByConnectedAccountId( + connectedAccountId, + workspaceId, + ); + + if (!calendarChannels || calendarChannels.length === 0) { + throw new Error( + `No calendar channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`, + ); + } + + return calendarChannels[0]; + } + + public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail( + connectedAccountId: string, + workspaceId: string, + ): Promise { + const calendarChannel = await this.getFirstByConnectedAccountIdOrFail( + connectedAccountId, + workspaceId, + ); + + return calendarChannel.isContactAutoCreationEnabled; + } + + public async getByIds( + ids: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarChannel" WHERE "id" = ANY($1)`, + [ids], + workspaceId, + transactionManager, + ); + } +} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module.ts new file mode 100644 index 000000000..87a522968 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; + +import { CalendarEventAttendeeService } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service'; +import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [WorkspaceDataSourceModule], + providers: [CalendarEventAttendeeService], + exports: [CalendarEventAttendeeService], +}) +export class CalendarEventAttendeeModule {} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service.ts new file mode 100644 index 000000000..09f04d891 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service.ts @@ -0,0 +1,151 @@ +import { Injectable } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service'; +import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record'; +import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata'; +import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util'; +import { CalendarEventAttendee } from 'src/workspace/calendar/types/calendar-event'; + +@Injectable() +export class CalendarEventAttendeeService { + constructor( + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) {} + + public async getByIds( + calendarEventAttendeeIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + if (calendarEventAttendeeIds.length === 0) { + return []; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`, + [calendarEventAttendeeIds], + workspaceId, + transactionManager, + ); + } + + public async getByCalendarEventIds( + calendarEventIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + if (calendarEventIds.length === 0) { + return []; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "calendarEventId" = ANY($1)`, + [calendarEventIds], + workspaceId, + transactionManager, + ); + } + + public async deleteByIds( + calendarEventAttendeeIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise { + if (calendarEventAttendeeIds.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + await this.workspaceDataSourceService.executeRawQuery( + `DELETE FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`, + [calendarEventAttendeeIds], + workspaceId, + transactionManager, + ); + } + + public async saveCalendarEventAttendees( + calendarEventAttendees: CalendarEventAttendee[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise { + if (calendarEventAttendees.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const { flattenedValues, valuesString } = + getFlattenedValuesAndValuesStringForBatchRawQuery( + calendarEventAttendees, + { + calendarEventId: 'uuid', + handle: 'text', + displayName: 'text', + isOrganizer: 'boolean', + responseStatus: `${dataSourceSchema}."calendarEventAttendee_responsestatus_enum"`, + }, + ); + + await this.workspaceDataSourceService.executeRawQuery( + `INSERT INTO ${dataSourceSchema}."calendarEventAttendee" ("calendarEventId", "handle", "displayName", "isOrganizer", "responseStatus") VALUES ${valuesString}`, + flattenedValues, + workspaceId, + transactionManager, + ); + } + + public async updateCalendarEventAttendees( + calendarEventAttendees: CalendarEventAttendee[], + iCalUIDCalendarEventIdMap: Map, + workspaceId: string, + transactionManager?: EntityManager, + ): Promise { + if (calendarEventAttendees.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const values = calendarEventAttendees.map((calendarEventAttendee) => ({ + ...calendarEventAttendee, + calendarEventId: iCalUIDCalendarEventIdMap.get( + calendarEventAttendee.iCalUID, + ), + })); + + const { flattenedValues, valuesString } = + getFlattenedValuesAndValuesStringForBatchRawQuery(values, { + calendarEventId: 'uuid', + handle: 'text', + displayName: 'text', + isOrganizer: 'boolean', + responseStatus: `${dataSourceSchema}."calendarEventAttendee_responsestatus_enum"`, + }); + + await this.workspaceDataSourceService.executeRawQuery( + `UPDATE ${dataSourceSchema}."calendarEventAttendee" AS "calendarEventAttendee" + SET "displayName" = "newValues"."displayName", + "isOrganizer" = "newValues"."isOrganizer", + "responseStatus" = "newValues"."responseStatus" + FROM (VALUES ${valuesString}) AS "newValues"("calendarEventId", "handle", "displayName", "isOrganizer", "responseStatus") + WHERE "calendarEventAttendee"."handle" = "newValues"."handle" + AND "calendarEventAttendee"."calendarEventId" = "newValues"."calendarEventId"`, + flattenedValues, + workspaceId, + transactionManager, + ); + } +} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.module.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.module.ts new file mode 100644 index 000000000..8e2efc3b3 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; + +import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service'; +import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [WorkspaceDataSourceModule], + providers: [CalendarEventService], + exports: [CalendarEventService], +}) +export class CalendarEventModule {} diff --git a/packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.service.ts b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.service.ts new file mode 100644 index 000000000..f2b8f0099 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.service.ts @@ -0,0 +1,202 @@ +import { Injectable } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service'; +import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record'; +import { CalendarEventObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata'; +import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util'; +import { CalendarEvent } from 'src/workspace/calendar/types/calendar-event'; +import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata'; + +@Injectable() +export class CalendarEventService { + constructor( + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) {} + + public async getByIds( + calendarEventIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + if (calendarEventIds.length === 0) { + return []; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + return await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."calendarEvent" WHERE "id" = ANY($1)`, + [calendarEventIds], + workspaceId, + transactionManager, + ); + } + + public async deleteByIds( + calendarEventIds: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise { + if (calendarEventIds.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + await this.workspaceDataSourceService.executeRawQuery( + `DELETE FROM ${dataSourceSchema}."calendarEvent" WHERE "id" = ANY($1)`, + [calendarEventIds], + workspaceId, + transactionManager, + ); + } + + public async getNonAssociatedCalendarEventIdsPaginated( + limit: number, + offset: number, + workspaceId: string, + transactionManager?: EntityManager, + ): Promise[]> { + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const nonAssociatedCalendarEvents = + await this.workspaceDataSourceService.executeRawQuery( + `SELECT m.id FROM ${dataSourceSchema}."calendarEvent" m + LEFT JOIN ${dataSourceSchema}."calendarChannelEventAssociation" ccea + ON m.id = ccea."calendarEventId" + WHERE ccea.id IS NULL + LIMIT $1 OFFSET $2`, + [limit, offset], + workspaceId, + transactionManager, + ); + + return nonAssociatedCalendarEvents.map(({ id }) => id); + } + + public async getICalUIDCalendarEventIdMap( + iCalUIDs: string[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise> { + if (iCalUIDs.length === 0) { + return new Map(); + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const calendarEvents: { + id: string; + iCalUID: string; + }[] = await this.workspaceDataSourceService.executeRawQuery( + `SELECT id, "iCalUID" FROM ${dataSourceSchema}."calendarEvent" WHERE "iCalUID" = ANY($1)`, + [iCalUIDs], + workspaceId, + transactionManager, + ); + + const iCalUIDsCalendarEvnetIdsMap = new Map(); + + calendarEvents.forEach((calendarEvent) => { + iCalUIDsCalendarEvnetIdsMap.set(calendarEvent.iCalUID, calendarEvent.id); + }); + + return iCalUIDsCalendarEvnetIdsMap; + } + + public async saveCalendarEvents( + calendarEvents: CalendarEvent[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise { + if (calendarEvents.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const { flattenedValues, valuesString } = + getFlattenedValuesAndValuesStringForBatchRawQuery(calendarEvents, { + id: 'uuid', + title: 'text', + isCanceled: 'boolean', + isFullDay: 'boolean', + startsAt: 'timestamptz', + endsAt: 'timestamptz', + externalCreatedAt: 'timestamptz', + externalUpdatedAt: 'timestamptz', + description: 'text', + location: 'text', + iCalUID: 'text', + conferenceSolution: 'text', + conferenceUri: 'text', + recurringEventExternalId: 'text', + }); + + await this.workspaceDataSourceService.executeRawQuery( + `INSERT INTO ${dataSourceSchema}."calendarEvent" ("id", "title", "isCanceled", "isFullDay", "startsAt", "endsAt", "externalCreatedAt", "externalUpdatedAt", "description", "location", "iCalUID", "conferenceSolution", "conferenceUri", "recurringEventExternalId") VALUES ${valuesString}`, + flattenedValues, + workspaceId, + transactionManager, + ); + } + + public async updateCalendarEvents( + calendarEvents: CalendarEvent[], + workspaceId: string, + transactionManager?: EntityManager, + ): Promise { + if (calendarEvents.length === 0) { + return; + } + + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const { flattenedValues, valuesString } = + getFlattenedValuesAndValuesStringForBatchRawQuery(calendarEvents, { + title: 'text', + isCanceled: 'boolean', + isFullDay: 'boolean', + startsAt: 'timestamptz', + endsAt: 'timestamptz', + externalCreatedAt: 'timestamptz', + externalUpdatedAt: 'timestamptz', + description: 'text', + location: 'text', + iCalUID: 'text', + conferenceSolution: 'text', + conferenceUri: 'text', + recurringEventExternalId: 'text', + }); + + await this.workspaceDataSourceService.executeRawQuery( + `UPDATE ${dataSourceSchema}."calendarEvent" AS "calendarEvent" + SET "title" = "newData"."title", + "isCanceled" = "newData"."isCanceled", + "isFullDay" = "newData"."isFullDay", + "startsAt" = "newData"."startsAt", + "endsAt" = "newData"."endsAt", + "externalCreatedAt" = "newData"."externalCreatedAt", + "externalUpdatedAt" = "newData"."externalUpdatedAt", + "description" = "newData"."description", + "location" = "newData"."location", + "conferenceSolution" = "newData"."conferenceSolution", + "conferenceUri" = "newData"."conferenceUri", + "recurringEventExternalId" = "newData"."recurringEventExternalId" + FROM (VALUES ${valuesString}) + AS "newData"("title", "isCanceled", "isFullDay", "startsAt", "endsAt", "externalCreatedAt", "externalUpdatedAt", "description", "location", "iCalUID", "conferenceSolution", "conferenceUri", "recurringEventExternalId") + WHERE "calendarEvent"."iCalUID" = "newData"."iCalUID"`, + flattenedValues, + workspaceId, + transactionManager, + ); + } +} diff --git a/packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module.ts b/packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module.ts new file mode 100644 index 000000000..01a9b56bd --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; + +import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; +import { DataSourceModule } from 'src/metadata/data-source/data-source.module'; +import { CalendarEventModule } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.module'; +import { CalendarEventCleanerService } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service'; + +@Module({ + imports: [DataSourceModule, TypeORMModule, CalendarEventModule], + providers: [CalendarEventCleanerService], + exports: [CalendarEventCleanerService], +}) +export class CalendarEventCleanerModule {} diff --git a/packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service.ts b/packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service.ts new file mode 100644 index 000000000..913880d0f --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service.ts @@ -0,0 +1,20 @@ +import { Injectable } from '@nestjs/common'; + +import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service'; +import { deleteUsingPagination } from 'src/workspace/messaging/services/thread-cleaner/utils/delete-using-pagination.util'; + +@Injectable() +export class CalendarEventCleanerService { + constructor(private readonly calendarEventService: CalendarEventService) {} + + public async cleanWorkspaceCalendarEvents(workspaceId: string) { + await deleteUsingPagination( + workspaceId, + 500, + this.calendarEventService.getNonAssociatedCalendarEventIdsPaginated.bind( + this.calendarEventService, + ), + this.calendarEventService.deleteByIds.bind(this.calendarEventService), + ); + } +} diff --git a/packages/twenty-server/src/workspace/calendar/services/google-calendar-full-sync.service.ts b/packages/twenty-server/src/workspace/calendar/services/google-calendar-full-sync.service.ts new file mode 100644 index 000000000..6a52ae9f5 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/services/google-calendar-full-sync.service.ts @@ -0,0 +1,274 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; +import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service'; +import { + FeatureFlagEntity, + FeatureFlagKeys, +} from 'src/core/feature-flag/feature-flag.entity'; +import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider'; +import { googleCalendarSearchFilterExcludeEmails } from 'src/workspace/calendar/utils/google-calendar-search-filter.util'; +import { CalendarChannelEventAssociationService } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service'; +import { CalendarChannelService } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.service'; +import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service'; +import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service'; +import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service'; +import { formatGoogleCalendarEvent } from 'src/workspace/calendar/utils/format-google-calendar-event.util'; +import { GoogleCalendarFullSyncJobData } from 'src/workspace/calendar/jobs/google-calendar-full-sync.job'; +import { CalendarEventAttendeeService } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service'; + +@Injectable() +export class GoogleCalendarFullSyncService { + private readonly logger = new Logger(GoogleCalendarFullSyncService.name); + + constructor( + private readonly googleCalendarClientProvider: GoogleCalendarClientProvider, + @Inject(MessageQueue.calendarQueue) + private readonly messageQueueService: MessageQueueService, + private readonly connectedAccountService: ConnectedAccountService, + private readonly calendarEventService: CalendarEventService, + private readonly calendarChannelService: CalendarChannelService, + private readonly calendarChannelEventAssociationService: CalendarChannelEventAssociationService, + private readonly calendarEventAttendeesService: CalendarEventAttendeeService, + private readonly blocklistService: BlocklistService, + @InjectRepository(FeatureFlagEntity, 'core') + private readonly featureFlagRepository: Repository, + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) {} + + public async startGoogleCalendarFullSync( + workspaceId: string, + connectedAccountId: string, + pageToken?: string, + ): Promise { + const connectedAccount = await this.connectedAccountService.getById( + connectedAccountId, + workspaceId, + ); + + if (!connectedAccount) { + return; + } + + const refreshToken = connectedAccount.refreshToken; + const workspaceMemberId = connectedAccount.accountOwnerId; + + if (!refreshToken) { + throw new Error( + `No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`, + ); + } + + const calendarChannel = + await this.calendarChannelService.getFirstByConnectedAccountIdOrFail( + connectedAccountId, + workspaceId, + ); + + const calendarChannelId = calendarChannel.id; + + const googleCalendarClient = + await this.googleCalendarClientProvider.getGoogleCalendarClient( + refreshToken, + ); + + const isBlocklistEnabledFeatureFlag = + await this.featureFlagRepository.findOneBy({ + workspaceId, + key: FeatureFlagKeys.IsBlocklistEnabled, + value: true, + }); + + const isBlocklistEnabled = + isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value; + + const blocklist = isBlocklistEnabled + ? await this.blocklistService.getByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ) + : []; + + const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle); + let startTime = Date.now(); + + const googleCalendarEvents = await googleCalendarClient.events.list({ + calendarId: 'primary', + maxResults: 500, + pageToken: pageToken, + q: googleCalendarSearchFilterExcludeEmails(blocklistedEmails), + }); + + let endTime = Date.now(); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} getting events list in ${ + endTime - startTime + }ms.`, + ); + + const { + items: events, + nextPageToken, + nextSyncToken, + } = googleCalendarEvents.data; + + if (!events || events?.length === 0) { + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`, + ); + + return; + } + + const eventExternalIds = events.map((event) => event.id as string); + + startTime = Date.now(); + + const existingCalendarChannelEventAssociations = + await this.calendarChannelEventAssociationService.getByEventExternalIdsAndCalendarChannelId( + eventExternalIds, + calendarChannelId, + workspaceId, + ); + + endTime = Date.now(); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: getting existing calendar channel event associations in ${ + endTime - startTime + }ms.`, + ); + + // TODO: In V2, we will also import deleted events by doing batch GET queries on the canceled events + // The canceled events start and end are not accessible in the list query + const formattedEvents = events + .filter((event) => event.status !== 'cancelled') + .map((event) => formatGoogleCalendarEvent(event)); + + // TODO: When we will be able to add unicity contraint on iCalUID, we will do a INSERT ON CONFLICT DO UPDATE + + const existingEventExternalIds = + existingCalendarChannelEventAssociations.map( + (association) => association.eventExternalId, + ); + + const eventsToSave = formattedEvents.filter( + (event) => !existingEventExternalIds.includes(event.externalId), + ); + + const eventsToUpdate = formattedEvents.filter((event) => + existingEventExternalIds.includes(event.externalId), + ); + + const calendarChannelEventAssociationsToSave = eventsToSave.map( + (event) => ({ + calendarEventId: event.id, + eventExternalId: event.externalId, + calendarChannelId, + }), + ); + + const attendeesToSave = eventsToSave.flatMap((event) => event.attendees); + + const attendeesToUpdate = eventsToUpdate.flatMap( + (event) => event.attendees, + ); + + const iCalUIDCalendarEventIdMap = + await this.calendarEventService.getICalUIDCalendarEventIdMap( + eventsToUpdate.map((event) => event.iCalUID), + workspaceId, + ); + + if (events.length > 0) { + const dataSourceMetadata = + await this.workspaceDataSourceService.connectToWorkspaceDataSource( + workspaceId, + ); + + dataSourceMetadata?.transaction(async (transactionManager) => { + this.calendarEventService.saveCalendarEvents( + eventsToSave, + workspaceId, + transactionManager, + ); + + this.calendarEventService.updateCalendarEvents( + eventsToUpdate, + workspaceId, + transactionManager, + ); + + this.calendarChannelEventAssociationService.saveCalendarChannelEventAssociations( + calendarChannelEventAssociationsToSave, + workspaceId, + transactionManager, + ); + + this.calendarEventAttendeesService.saveCalendarEventAttendees( + attendeesToSave, + workspaceId, + transactionManager, + ); + + this.calendarEventAttendeesService.updateCalendarEventAttendees( + attendeesToUpdate, + iCalUIDCalendarEventIdMap, + workspaceId, + transactionManager, + ); + }); + } else { + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`, + ); + } + + if (!nextSyncToken) { + throw new Error( + `No next sync token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`, + ); + } + + startTime = Date.now(); + + // await this.calendarChannelService.updateSyncCursor( + // nextSyncToken, + // connectedAccount.id, + // workspaceId, + // ); + + endTime = Date.now(); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating sync cursor in ${ + endTime - startTime + }ms.`, + ); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} ${ + nextPageToken ? `and ${nextPageToken} pageToken` : '' + } done.`, + ); + + if (nextPageToken) { + await this.messageQueueService.add( + GoogleCalendarFullSyncService.name, + { + workspaceId, + connectedAccountId, + nextPageToken, + }, + { + retryLimit: 2, + }, + ); + } + } +} diff --git a/packages/twenty-server/src/workspace/calendar/services/providers/calendar-providers.module.ts b/packages/twenty-server/src/workspace/calendar/services/providers/calendar-providers.module.ts new file mode 100644 index 000000000..adc8b9069 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/services/providers/calendar-providers.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; + +import { EnvironmentModule } from 'src/integrations/environment/environment.module'; +import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider'; + +@Module({ + imports: [EnvironmentModule], + providers: [GoogleCalendarClientProvider], + exports: [GoogleCalendarClientProvider], +}) +export class CalendarProvidersModule {} diff --git a/packages/twenty-server/src/workspace/calendar/services/providers/google-calendar/google-calendar.provider.ts b/packages/twenty-server/src/workspace/calendar/services/providers/google-calendar/google-calendar.provider.ts new file mode 100644 index 000000000..15b157a43 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/services/providers/google-calendar/google-calendar.provider.ts @@ -0,0 +1,42 @@ +import { Injectable } from '@nestjs/common'; + +import { OAuth2Client } from 'google-auth-library'; +import { calendar_v3 as calendarV3, google } from 'googleapis'; + +import { EnvironmentService } from 'src/integrations/environment/environment.service'; + +@Injectable() +export class GoogleCalendarClientProvider { + constructor(private readonly environmentService: EnvironmentService) {} + + public async getGoogleCalendarClient( + refreshToken: string, + ): Promise { + const oAuth2Client = await this.getOAuth2Client(refreshToken); + + const googleCalendarClient = google.calendar({ + version: 'v3', + auth: oAuth2Client, + }); + + return googleCalendarClient; + } + + private async getOAuth2Client(refreshToken: string): Promise { + const googleCalendarClientId = + this.environmentService.getAuthGoogleClientId(); + const googleCalendarClientSecret = + this.environmentService.getAuthGoogleClientSecret(); + + const oAuth2Client = new google.auth.OAuth2( + googleCalendarClientId, + googleCalendarClientSecret, + ); + + oAuth2Client.setCredentials({ + refresh_token: refreshToken, + }); + + return oAuth2Client; + } +} diff --git a/packages/twenty-server/src/workspace/calendar/types/calendar-event.ts b/packages/twenty-server/src/workspace/calendar/types/calendar-event.ts new file mode 100644 index 000000000..06c8a63a7 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/types/calendar-event.ts @@ -0,0 +1,31 @@ +import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata'; +import { CalendarEventObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata'; +import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record'; + +export type CalendarEvent = Omit< + ObjectRecord, + | 'createdAt' + | 'updatedAt' + | 'calendarChannelEventAssociations' + | 'calendarEventAttendees' + | 'eventAttendees' +>; + +export type CalendarEventAttendee = Omit< + ObjectRecord, + | 'id' + | 'createdAt' + | 'updatedAt' + | 'personId' + | 'workspaceMemberId' + | 'person' + | 'workspaceMember' + | 'calendarEvent' +> & { + iCalUID: string; +}; + +export type CalendarEventWithAttendees = CalendarEvent & { + externalId: string; + attendees: CalendarEventAttendee[]; +}; diff --git a/packages/twenty-server/src/workspace/calendar/utils/format-google-calendar-event.util.ts b/packages/twenty-server/src/workspace/calendar/utils/format-google-calendar-event.util.ts new file mode 100644 index 000000000..29f4878b9 --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/utils/format-google-calendar-event.util.ts @@ -0,0 +1,52 @@ +import { calendar_v3 } from 'googleapis'; +import { v4 } from 'uuid'; + +import { CalendarEventWithAttendees } from 'src/workspace/calendar/types/calendar-event'; +import { CalendarEventAttendeeResponseStatus } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata'; + +export const formatGoogleCalendarEvent = ( + event: calendar_v3.Schema$Event, +): CalendarEventWithAttendees => { + const id = v4(); + + const formatResponseStatus = (status: string | null | undefined) => { + switch (status) { + case 'accepted': + return CalendarEventAttendeeResponseStatus.ACCEPTED; + case 'declined': + return CalendarEventAttendeeResponseStatus.DECLINED; + case 'tentative': + return CalendarEventAttendeeResponseStatus.TENTATIVE; + default: + return CalendarEventAttendeeResponseStatus.NEEDS_ACTION; + } + }; + + return { + id, + title: event.summary ?? '', + isCanceled: event.status === 'cancelled', + isFullDay: event.start?.dateTime == null, + startsAt: event.start?.dateTime ?? event.start?.date ?? null, + endsAt: event.end?.dateTime ?? event.end?.date ?? null, + externalId: event.id ?? '', + externalCreatedAt: event.created ?? null, + externalUpdatedAt: event.updated ?? null, + description: event.description ?? '', + location: event.location ?? '', + iCalUID: event.iCalUID ?? '', + conferenceSolution: + event.conferenceData?.conferenceSolution?.key?.type ?? '', + conferenceUri: event.conferenceData?.entryPoints?.[0]?.uri ?? '', + recurringEventExternalId: event.recurringEventId ?? '', + attendees: + event.attendees?.map((attendee) => ({ + calendarEventId: id, + iCalUID: event.iCalUID ?? '', + handle: attendee.email ?? '', + displayName: attendee.displayName ?? '', + isOrganizer: attendee.organizer === true, + responseStatus: formatResponseStatus(attendee.responseStatus), + })) ?? [], + }; +}; diff --git a/packages/twenty-server/src/workspace/calendar/utils/google-calendar-search-filter.util.ts b/packages/twenty-server/src/workspace/calendar/utils/google-calendar-search-filter.util.ts new file mode 100644 index 000000000..c9c90417a --- /dev/null +++ b/packages/twenty-server/src/workspace/calendar/utils/google-calendar-search-filter.util.ts @@ -0,0 +1,9 @@ +export const googleCalendarSearchFilterExcludeEmails = ( + emails: string[], +): string => { + if (emails.length === 0) { + return ''; + } + + return `email=-${emails.join(', -')}`; +}; diff --git a/packages/twenty-server/src/workspace/messaging/commands/crons/fetch-all-workspaces-messages.job.ts b/packages/twenty-server/src/workspace/messaging/commands/crons/fetch-all-workspaces-messages.job.ts index 9d02f2705..b0b82728d 100644 --- a/packages/twenty-server/src/workspace/messaging/commands/crons/fetch-all-workspaces-messages.job.ts +++ b/packages/twenty-server/src/workspace/messaging/commands/crons/fetch-all-workspaces-messages.job.ts @@ -7,7 +7,7 @@ import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/messa import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; import { Workspace } from 'src/core/workspace/workspace.entity'; import { GmailPartialSyncJobData, diff --git a/packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts b/packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts index 9e1dba862..d7ff7212d 100644 --- a/packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts +++ b/packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts @@ -6,7 +6,7 @@ import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { DataSourceModule } from 'src/metadata/data-source/data-source.module'; import { GmailFullSyncCommand } from 'src/workspace/messaging/commands/gmail-full-sync.command'; import { GmailPartialSyncCommand } from 'src/workspace/messaging/commands/gmail-partial-sync.command'; -import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module'; +import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module'; import { StartFetchAllWorkspacesMessagesCronCommand } from 'src/workspace/messaging/commands/start-fetch-all-workspaces-messages.cron.command'; import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/workspace/messaging/commands/stop-fetch-all-workspaces-messages.cron.command'; diff --git a/packages/twenty-server/src/workspace/messaging/commands/gmail-full-sync.command.ts b/packages/twenty-server/src/workspace/messaging/commands/gmail-full-sync.command.ts index 586570070..b9a54a73d 100644 --- a/packages/twenty-server/src/workspace/messaging/commands/gmail-full-sync.command.ts +++ b/packages/twenty-server/src/workspace/messaging/commands/gmail-full-sync.command.ts @@ -8,7 +8,7 @@ import { GmailFullSyncJobData, GmailFullSyncJob, } from 'src/workspace/messaging/jobs/gmail-full-sync.job'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; interface GmailFullSyncOptions { workspaceId: string; diff --git a/packages/twenty-server/src/workspace/messaging/commands/gmail-partial-sync.command.ts b/packages/twenty-server/src/workspace/messaging/commands/gmail-partial-sync.command.ts index 134023630..482085ebb 100644 --- a/packages/twenty-server/src/workspace/messaging/commands/gmail-partial-sync.command.ts +++ b/packages/twenty-server/src/workspace/messaging/commands/gmail-partial-sync.command.ts @@ -8,7 +8,7 @@ import { GmailPartialSyncJob, GmailPartialSyncJobData, } from 'src/workspace/messaging/jobs/gmail-partial-sync.job'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; interface GmailPartialSyncOptions { workspaceId: string; diff --git a/packages/twenty-server/src/workspace/messaging/jobs/create-companies-and-contacts-after-sync.job.ts b/packages/twenty-server/src/workspace/messaging/jobs/create-companies-and-contacts-after-sync.job.ts index a599969d8..2326de4d7 100644 --- a/packages/twenty-server/src/workspace/messaging/jobs/create-companies-and-contacts-after-sync.job.ts +++ b/packages/twenty-server/src/workspace/messaging/jobs/create-companies-and-contacts-after-sync.job.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface'; -import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service'; +import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service'; import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service'; import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service'; @@ -19,7 +19,7 @@ export class CreateCompaniesAndContactsAfterSyncJob CreateCompaniesAndContactsAfterSyncJob.name, ); constructor( - private readonly createCompaniesAndContactsService: CreateCompaniesAndContactsService, + private readonly createCompaniesAndContactsService: CreateCompanyAndContactService, private readonly messageChannelService: MessageChannelService, private readonly messageParticipantService: MessageParticipantService, ) {} diff --git a/packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job.ts b/packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job.ts new file mode 100644 index 000000000..dbb9b0a05 --- /dev/null +++ b/packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job.ts @@ -0,0 +1,40 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface'; + +import { CalendarEventCleanerService } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service'; + +export type DeleteConnectedAccountAssociatedCalendarDataJobData = { + workspaceId: string; + connectedAccountId: string; +}; + +@Injectable() +export class DeleteConnectedAccountAssociatedCalendarDataJob + implements + MessageQueueJob +{ + private readonly logger = new Logger( + DeleteConnectedAccountAssociatedCalendarDataJob.name, + ); + + constructor( + private readonly calendarEventCleanerService: CalendarEventCleanerService, + ) {} + + async handle( + data: DeleteConnectedAccountAssociatedCalendarDataJobData, + ): Promise { + this.logger.log( + `Deleting connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`, + ); + + await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents( + data.workspaceId, + ); + + this.logger.log( + `Deleted connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`, + ); + } +} diff --git a/packages/twenty-server/src/workspace/messaging/jobs/delete-connected-acount-associated-data.job.ts b/packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job.ts similarity index 63% rename from packages/twenty-server/src/workspace/messaging/jobs/delete-connected-acount-associated-data.job.ts rename to packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job.ts index 81c5bb35c..55a8eb3fc 100644 --- a/packages/twenty-server/src/workspace/messaging/jobs/delete-connected-acount-associated-data.job.ts +++ b/packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job.ts @@ -4,32 +4,33 @@ import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/messa import { ThreadCleanerService } from 'src/workspace/messaging/services/thread-cleaner/thread-cleaner.service'; -export type DeleteConnectedAccountAssociatedDataJobData = { +export type DeleteConnectedAccountAssociatedMessagingDataJobData = { workspaceId: string; connectedAccountId: string; }; @Injectable() -export class DeleteConnectedAccountAssociatedDataJob - implements MessageQueueJob +export class DeleteConnectedAccountAssociatedMessagingDataJob + implements + MessageQueueJob { private readonly logger = new Logger( - DeleteConnectedAccountAssociatedDataJob.name, + DeleteConnectedAccountAssociatedMessagingDataJob.name, ); constructor(private readonly threadCleanerService: ThreadCleanerService) {} async handle( - data: DeleteConnectedAccountAssociatedDataJobData, + data: DeleteConnectedAccountAssociatedMessagingDataJobData, ): Promise { this.logger.log( - `Deleting connected account ${data.connectedAccountId} associated data in workspace ${data.workspaceId}`, + `Deleting connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`, ); await this.threadCleanerService.cleanWorkspaceThreads(data.workspaceId); this.logger.log( - `Deleted connected account ${data.connectedAccountId} associated data in workspace ${data.workspaceId}`, + `Deleted connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`, ); } } diff --git a/packages/twenty-server/src/workspace/messaging/jobs/gmail-full-sync.job.ts b/packages/twenty-server/src/workspace/messaging/jobs/gmail-full-sync.job.ts index 764ba2f62..65a7d96c8 100644 --- a/packages/twenty-server/src/workspace/messaging/jobs/gmail-full-sync.job.ts +++ b/packages/twenty-server/src/workspace/messaging/jobs/gmail-full-sync.job.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface'; -import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service'; +import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service'; import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service'; export type GmailFullSyncJobData = { @@ -16,7 +16,7 @@ export class GmailFullSyncJob implements MessageQueueJob { private readonly logger = new Logger(GmailFullSyncJob.name); constructor( - private readonly gmailRefreshAccessTokenService: GmailRefreshAccessTokenService, + private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService, private readonly gmailFullSyncService: GmailFullSyncService, ) {} @@ -28,7 +28,7 @@ export class GmailFullSyncJob implements MessageQueueJob { ); try { - await this.gmailRefreshAccessTokenService.refreshAndSaveAccessToken( + await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken( data.workspaceId, data.connectedAccountId, ); diff --git a/packages/twenty-server/src/workspace/messaging/jobs/gmail-partial-sync.job.ts b/packages/twenty-server/src/workspace/messaging/jobs/gmail-partial-sync.job.ts index 057b5f762..cfafb37c1 100644 --- a/packages/twenty-server/src/workspace/messaging/jobs/gmail-partial-sync.job.ts +++ b/packages/twenty-server/src/workspace/messaging/jobs/gmail-partial-sync.job.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface'; -import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service'; +import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service'; import { GmailPartialSyncService } from 'src/workspace/messaging/services/gmail-partial-sync.service'; export type GmailPartialSyncJobData = { @@ -17,7 +17,7 @@ export class GmailPartialSyncJob private readonly logger = new Logger(GmailPartialSyncJob.name); constructor( - private readonly gmailRefreshAccessTokenService: GmailRefreshAccessTokenService, + private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService, private readonly gmailPartialSyncService: GmailPartialSyncService, ) {} @@ -27,7 +27,7 @@ export class GmailPartialSyncJob ); try { - await this.gmailRefreshAccessTokenService.refreshAndSaveAccessToken( + await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken( data.workspaceId, data.connectedAccountId, ); diff --git a/packages/twenty-server/src/workspace/messaging/listeners/messaging-connected-account.listener.ts b/packages/twenty-server/src/workspace/messaging/listeners/messaging-connected-account.listener.ts index 1aeee7dd8..baef313da 100644 --- a/packages/twenty-server/src/workspace/messaging/listeners/messaging-connected-account.listener.ts +++ b/packages/twenty-server/src/workspace/messaging/listeners/messaging-connected-account.listener.ts @@ -5,9 +5,13 @@ import { ObjectRecordDeleteEvent } from 'src/integrations/event-emitter/types/ob import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service'; import { - DeleteConnectedAccountAssociatedDataJobData, - DeleteConnectedAccountAssociatedDataJob, -} from 'src/workspace/messaging/jobs/delete-connected-acount-associated-data.job'; + DeleteConnectedAccountAssociatedCalendarDataJobData, + DeleteConnectedAccountAssociatedCalendarDataJob, +} from 'src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job'; +import { + DeleteConnectedAccountAssociatedMessagingDataJobData, + DeleteConnectedAccountAssociatedMessagingDataJob, +} from 'src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job'; import { ConnectedAccountObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/connected-account.object-metadata'; @Injectable() @@ -15,14 +19,24 @@ export class MessagingConnectedAccountListener { constructor( @Inject(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, + @Inject(MessageQueue.calendarQueue) + private readonly calendarQueueService: MessageQueueService, ) {} @OnEvent('connectedAccount.deleted') handleDeletedEvent( payload: ObjectRecordDeleteEvent, ) { - this.messageQueueService.add( - DeleteConnectedAccountAssociatedDataJob.name, + this.messageQueueService.add( + DeleteConnectedAccountAssociatedMessagingDataJob.name, + { + workspaceId: payload.workspaceId, + connectedAccountId: payload.deletedRecord.id, + }, + ); + + this.calendarQueueService.add( + DeleteConnectedAccountAssociatedCalendarDataJob.name, { workspaceId: payload.workspaceId, connectedAccountId: payload.deletedRecord.id, diff --git a/packages/twenty-server/src/workspace/messaging/messaging.module.ts b/packages/twenty-server/src/workspace/messaging/messaging.module.ts index b15a8b444..a15f7fe12 100644 --- a/packages/twenty-server/src/workspace/messaging/messaging.module.ts +++ b/packages/twenty-server/src/workspace/messaging/messaging.module.ts @@ -1,7 +1,8 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { HttpModule } from '@nestjs/axios'; -import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module'; +import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module'; import { MessageChannelMessageAssociationModule } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-assocation.module'; import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module'; import { MessageThreadModule } from 'src/workspace/messaging/repositories/message-thread/message-thread.module'; @@ -9,25 +10,26 @@ import { EnvironmentModule } from 'src/integrations/environment/environment.modu import { MessagingPersonListener } from 'src/workspace/messaging/listeners/messaging-person.listener'; import { MessageModule } from 'src/workspace/messaging/repositories/message/message.module'; import { GmailClientProvider } from 'src/workspace/messaging/services/providers/gmail/gmail-client.provider'; -import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service'; -import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service'; +import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service'; +import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service'; import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service'; import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service'; import { GmailPartialSyncService } from 'src/workspace/messaging/services/gmail-partial-sync.service'; -import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service'; +import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; import { MessageParticipantModule } from 'src/workspace/messaging/repositories/message-participant/message-participant.module'; import { MessagingWorkspaceMemberListener } from 'src/workspace/messaging/listeners/messaging-workspace-member.listener'; import { MessagingMessageChannelListener } from 'src/workspace/messaging/listeners/messaging-message-channel.listener'; import { MessageService } from 'src/workspace/messaging/repositories/message/message.service'; -import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module'; +import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module'; import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity'; -import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module'; +import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module'; import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module'; -import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module'; +import { PersonModule } from 'src/workspace/repositories/person/person.module'; import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service'; import { MessagingConnectedAccountListener } from 'src/workspace/messaging/listeners/messaging-connected-account.listener'; -import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/blocklist.module'; +import { BlocklistModule } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module'; +import { FetchByBatchesService } from 'src/workspace/messaging/services/fetch-by-batch.service'; @Module({ imports: [ EnvironmentModule, @@ -44,12 +46,15 @@ import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/ CompanyModule, PersonModule, BlocklistModule, + HttpModule.register({ + baseURL: 'https://www.googleapis.com/batch/gmail/v1', + }), ], providers: [ GmailFullSyncService, GmailPartialSyncService, FetchMessagesByBatchesService, - GmailRefreshAccessTokenService, + GoogleAPIsRefreshAccessTokenService, GmailClientProvider, CreateContactService, CreateCompanyService, @@ -59,11 +64,13 @@ import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/ MessageService, SaveMessagesAndCreateContactsService, MessagingConnectedAccountListener, + FetchByBatchesService, ], exports: [ GmailPartialSyncService, GmailFullSyncService, - GmailRefreshAccessTokenService, + GoogleAPIsRefreshAccessTokenService, + FetchByBatchesService, ], }) export class MessagingModule {} diff --git a/packages/twenty-server/src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook.ts b/packages/twenty-server/src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook.ts index b5da50554..09d90da96 100644 --- a/packages/twenty-server/src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook.ts +++ b/packages/twenty-server/src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook.ts @@ -12,8 +12,8 @@ import { FindManyResolverArgs } from 'src/workspace/workspace-resolver-builder/i import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-association.service'; import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; -import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; +import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service'; @Injectable() export class MessageFindManyPreQueryHook implements WorkspacePreQueryHook { diff --git a/packages/twenty-server/src/workspace/messaging/query-hooks/messaging-query-hook.module.ts b/packages/twenty-server/src/workspace/messaging/query-hooks/messaging-query-hook.module.ts index daa52af28..8c1cff07e 100644 --- a/packages/twenty-server/src/workspace/messaging/query-hooks/messaging-query-hook.module.ts +++ b/packages/twenty-server/src/workspace/messaging/query-hooks/messaging-query-hook.module.ts @@ -2,10 +2,10 @@ import { Module } from '@nestjs/common'; import { MessageFindManyPreQueryHook } from 'src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook'; import { MessageFindOnePreQueryHook } from 'src/workspace/messaging/query-hooks/message/message-find-one.pre-query-hook'; -import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module'; +import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module'; import { MessageChannelMessageAssociationModule } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-assocation.module'; import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module'; -import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module'; +import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module'; @Module({ imports: [ diff --git a/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.module.ts b/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.module.ts index 9f924dc41..dc76061c1 100644 --- a/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.module.ts +++ b/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; -import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module'; +import { PersonModule } from 'src/workspace/repositories/person/person.module'; @Module({ imports: [WorkspaceDataSourceModule, PersonModule], diff --git a/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.service.ts b/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.service.ts index aebbf3a3a..ac08fb612 100644 --- a/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.service.ts +++ b/packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.service.ts @@ -9,7 +9,7 @@ import { ParticipantWithId, ParticipantWithMessageId, } from 'src/workspace/messaging/types/gmail-message'; -import { PersonService } from 'src/workspace/messaging/repositories/person/person.service'; +import { PersonService } from 'src/workspace/repositories/person/person.service'; @Injectable() export class MessageParticipantService { diff --git a/packages/twenty-server/src/workspace/messaging/repositories/message/message.module.ts b/packages/twenty-server/src/workspace/messaging/repositories/message/message.module.ts index 202352818..707e89cfe 100644 --- a/packages/twenty-server/src/workspace/messaging/repositories/message/message.module.ts +++ b/packages/twenty-server/src/workspace/messaging/repositories/message/message.module.ts @@ -6,7 +6,7 @@ import { MessageParticipantModule } from 'src/workspace/messaging/repositories/m import { MessageThreadModule } from 'src/workspace/messaging/repositories/message-thread/message-thread.module'; import { MessageService } from 'src/workspace/messaging/repositories/message/message.service'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; -import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module'; +import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module'; @Module({ imports: [ diff --git a/packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module.ts b/packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module.ts deleted file mode 100644 index 2baba41f0..000000000 --- a/packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module'; -import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module'; -import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service'; -import { CreateCompanyModule } from 'src/workspace/messaging/services/create-company/create-company.module'; -import { CreateContactModule } from 'src/workspace/messaging/services/create-contact/create-contact.module'; -import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; - -@Module({ - imports: [ - WorkspaceDataSourceModule, - CreateContactModule, - CreateCompanyModule, - WorkspaceMemberModule, - PersonModule, - ], - providers: [CreateCompaniesAndContactsService], - exports: [CreateCompaniesAndContactsService], -}) -export class CreateCompaniesAndContactsModule {} diff --git a/packages/twenty-server/src/workspace/messaging/services/fetch-by-batch.service.ts b/packages/twenty-server/src/workspace/messaging/services/fetch-by-batch.service.ts new file mode 100644 index 000000000..d64af2e57 --- /dev/null +++ b/packages/twenty-server/src/workspace/messaging/services/fetch-by-batch.service.ts @@ -0,0 +1,128 @@ +import { HttpService } from '@nestjs/axios'; +import { Injectable } from '@nestjs/common'; + +import { AxiosResponse } from 'axios'; + +import { BatchQueries } from 'src/workspace/calendar-and-messaging/types/batch-queries'; +import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmail-message-parsed-response'; + +@Injectable() +export class FetchByBatchesService { + constructor(private readonly httpService: HttpService) {} + + async fetchAllByBatches( + queries: BatchQueries, + accessToken: string, + boundary: string, + ): Promise[]> { + const batchLimit = 50; + + let batchOffset = 0; + + let batchResponses: AxiosResponse[] = []; + + while (batchOffset < queries.length) { + const batchResponse = await this.fetchBatch( + queries, + accessToken, + batchOffset, + batchLimit, + boundary, + ); + + batchResponses = batchResponses.concat(batchResponse); + + batchOffset += batchLimit; + } + + return batchResponses; + } + + async fetchBatch( + queries: BatchQueries, + accessToken: string, + batchOffset: number, + batchLimit: number, + boundary: string, + ): Promise> { + const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit); + + const response = await this.httpService.axiosRef.post( + '/', + this.createBatchBody(limitedQueries, boundary), + { + headers: { + 'Content-Type': 'multipart/mixed; boundary=' + boundary, + Authorization: 'Bearer ' + accessToken, + }, + }, + ); + + return response; + } + + createBatchBody(queries: BatchQueries, boundary: string): string { + let batchBody: string[] = []; + + queries.forEach(function (call) { + const method = 'GET'; + const uri = call.uri; + + batchBody = batchBody.concat([ + '--', + boundary, + '\r\n', + 'Content-Type: application/http', + '\r\n\r\n', + + method, + ' ', + uri, + '\r\n\r\n', + ]); + }); + + return batchBody.concat(['--', boundary, '--']).join(''); + } + + parseBatch( + responseCollection: AxiosResponse, + ): GmailMessageParsedResponse[] { + const responseItems: GmailMessageParsedResponse[] = []; + + const boundary = this.getBatchSeparator(responseCollection); + + const responseLines: string[] = responseCollection.data.split( + '--' + boundary, + ); + + responseLines.forEach(function (response) { + const startJson = response.indexOf('{'); + const endJson = response.lastIndexOf('}'); + + if (startJson < 0 || endJson < 0) return; + + const responseJson = response.substring(startJson, endJson + 1); + + const item = JSON.parse(responseJson); + + responseItems.push(item); + }); + + return responseItems; + } + + getBatchSeparator(responseCollection: AxiosResponse): string { + const headers = responseCollection.headers; + + const contentType: string = headers['content-type']; + + if (!contentType) return ''; + + const components = contentType.split('; '); + + const boundary = components.find((item) => item.startsWith('boundary=')); + + return boundary?.replace('boundary=', '').trim() || ''; + } +} diff --git a/packages/twenty-server/src/workspace/messaging/services/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/workspace/messaging/services/fetch-messages-by-batches.service.ts index a88ef5db6..f9186479f 100644 --- a/packages/twenty-server/src/workspace/messaging/services/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/workspace/messaging/services/fetch-messages-by-batches.service.ts @@ -1,6 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; -import axios, { AxiosInstance, AxiosResponse } from 'axios'; +import { AxiosResponse } from 'axios'; import { simpleParser, AddressObject } from 'mailparser'; import planer from 'planer'; @@ -10,17 +10,13 @@ import { } from 'src/workspace/messaging/types/gmail-message'; import { MessageQuery } from 'src/workspace/messaging/types/message-or-thread-query'; import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmail-message-parsed-response'; +import { FetchByBatchesService } from 'src/workspace/messaging/services/fetch-by-batch.service'; @Injectable() export class FetchMessagesByBatchesService { - private readonly httpService: AxiosInstance; private readonly logger = new Logger(FetchMessagesByBatchesService.name); - constructor() { - this.httpService = axios.create({ - baseURL: 'https://www.googleapis.com/batch/gmail/v1', - }); - } + constructor(private readonly fetchByBatchesService: FetchByBatchesService) {} async fetchAllMessages( queries: MessageQuery[], @@ -30,7 +26,7 @@ export class FetchMessagesByBatchesService { connectedAccountId?: string, ): Promise<{ messages: GmailMessage[]; errors: any[] }> { let startTime = Date.now(); - const batchResponses = await this.fetchAllByBatches( + const batchResponses = await this.fetchByBatchesService.fetchAllByBatches( queries, accessToken, 'batch_gmail_messages', @@ -59,126 +55,10 @@ export class FetchMessagesByBatchesService { return formattedResponse; } - async fetchAllByBatches( - queries: MessageQuery[], - accessToken: string, - boundary: string, - ): Promise[]> { - const batchLimit = 50; - - let batchOffset = 0; - - let batchResponses: AxiosResponse[] = []; - - while (batchOffset < queries.length) { - const batchResponse = await this.fetchBatch( - queries, - accessToken, - batchOffset, - batchLimit, - boundary, - ); - - batchResponses = batchResponses.concat(batchResponse); - - batchOffset += batchLimit; - } - - return batchResponses; - } - - async fetchBatch( - queries: MessageQuery[], - accessToken: string, - batchOffset: number, - batchLimit: number, - boundary: string, - ): Promise> { - const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit); - - const response = await this.httpService.post( - '/', - this.createBatchBody(limitedQueries, boundary), - { - headers: { - 'Content-Type': 'multipart/mixed; boundary=' + boundary, - Authorization: 'Bearer ' + accessToken, - }, - }, - ); - - return response; - } - - createBatchBody(queries: MessageQuery[], boundary: string): string { - let batchBody: string[] = []; - - queries.forEach(function (call) { - const method = 'GET'; - const uri = call.uri; - - batchBody = batchBody.concat([ - '--', - boundary, - '\r\n', - 'Content-Type: application/http', - '\r\n\r\n', - - method, - ' ', - uri, - '\r\n\r\n', - ]); - }); - - return batchBody.concat(['--', boundary, '--']).join(''); - } - - parseBatch( - responseCollection: AxiosResponse, - ): GmailMessageParsedResponse[] { - const responseItems: GmailMessageParsedResponse[] = []; - - const boundary = this.getBatchSeparator(responseCollection); - - const responseLines: string[] = responseCollection.data.split( - '--' + boundary, - ); - - responseLines.forEach(function (response) { - const startJson = response.indexOf('{'); - const endJson = response.lastIndexOf('}'); - - if (startJson < 0 || endJson < 0) return; - - const responseJson = response.substring(startJson, endJson + 1); - - const item = JSON.parse(responseJson); - - responseItems.push(item); - }); - - return responseItems; - } - - getBatchSeparator(responseCollection: AxiosResponse): string { - const headers = responseCollection.headers; - - const contentType: string = headers['content-type']; - - if (!contentType) return ''; - - const components = contentType.split('; '); - - const boundary = components.find((item) => item.startsWith('boundary=')); - - return boundary?.replace('boundary=', '').trim() || ''; - } - async formatBatchResponseAsGmailMessage( responseCollection: AxiosResponse, ): Promise<{ messages: GmailMessage[]; errors: any[] }> { - const parsedResponses = this.parseBatch( + const parsedResponses = this.fetchByBatchesService.parseBatch( responseCollection, ) as GmailMessageParsedResponse[]; diff --git a/packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts b/packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts index 9e37b317d..f0cc30c0d 100644 --- a/packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts +++ b/packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts @@ -11,12 +11,12 @@ import { GmailFullSyncJobData, GmailFullSyncJob, } from 'src/workspace/messaging/jobs/gmail-full-sync.job'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service'; import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-association.service'; import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util'; -import { gmailSearchFilterExcludeEmails } from 'src/workspace/messaging/utils/gmail-search-filter'; -import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service'; +import { gmailSearchFilterExcludeEmails } from 'src/workspace/messaging/utils/gmail-search-filter.util'; +import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service'; import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service'; import { FeatureFlagEntity, diff --git a/packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts b/packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts index 28cc43fa6..b91cf5819 100644 --- a/packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts +++ b/packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts @@ -12,13 +12,13 @@ import { GmailFullSyncJob, GmailFullSyncJobData, } from 'src/workspace/messaging/jobs/gmail-full-sync.job'; -import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service'; +import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service'; import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service'; import { MessageService } from 'src/workspace/messaging/repositories/message/message.service'; import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util'; import { GmailMessage } from 'src/workspace/messaging/types/gmail-message'; import { isPersonEmail } from 'src/workspace/messaging/utils/is-person-email.util'; -import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service'; +import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service'; import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service'; import { FeatureFlagEntity, diff --git a/packages/twenty-server/src/workspace/messaging/services/save-messages-and-create-contacts.service.ts b/packages/twenty-server/src/workspace/messaging/services/save-messages-and-create-contacts.service.ts index c47832ca6..654a0c9e1 100644 --- a/packages/twenty-server/src/workspace/messaging/services/save-messages-and-create-contacts.service.ts +++ b/packages/twenty-server/src/workspace/messaging/services/save-messages-and-create-contacts.service.ts @@ -5,7 +5,7 @@ import { EntityManager } from 'typeorm'; import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service'; import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service'; import { MessageService } from 'src/workspace/messaging/repositories/message/message.service'; -import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service'; +import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service'; import { GmailMessage, ParticipantWithMessageId, @@ -23,7 +23,7 @@ export class SaveMessagesAndCreateContactsService { constructor( private readonly messageService: MessageService, private readonly messageChannelService: MessageChannelService, - private readonly createCompaniesAndContactsService: CreateCompaniesAndContactsService, + private readonly createCompaniesAndContactsService: CreateCompanyAndContactService, private readonly messageParticipantService: MessageParticipantService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, ) {} diff --git a/packages/twenty-server/src/workspace/messaging/utils/gmail-search-filter.ts b/packages/twenty-server/src/workspace/messaging/utils/gmail-search-filter.util.ts similarity index 100% rename from packages/twenty-server/src/workspace/messaging/utils/gmail-search-filter.ts rename to packages/twenty-server/src/workspace/messaging/utils/gmail-search-filter.util.ts diff --git a/packages/twenty-server/src/workspace/messaging/repositories/person/person.module.ts b/packages/twenty-server/src/workspace/repositories/person/person.module.ts similarity index 67% rename from packages/twenty-server/src/workspace/messaging/repositories/person/person.module.ts rename to packages/twenty-server/src/workspace/repositories/person/person.module.ts index d74ef65b1..983d0f807 100644 --- a/packages/twenty-server/src/workspace/messaging/repositories/person/person.module.ts +++ b/packages/twenty-server/src/workspace/repositories/person/person.module.ts @@ -1,9 +1,8 @@ import { Module } from '@nestjs/common'; -import { PersonService } from 'src/workspace/messaging/repositories/person/person.service'; +import { PersonService } from 'src/workspace/repositories/person/person.service'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; -// TODO: Move outside of the messaging module @Module({ imports: [WorkspaceDataSourceModule], providers: [PersonService], diff --git a/packages/twenty-server/src/workspace/messaging/repositories/person/person.service.ts b/packages/twenty-server/src/workspace/repositories/person/person.service.ts similarity index 100% rename from packages/twenty-server/src/workspace/messaging/repositories/person/person.service.ts rename to packages/twenty-server/src/workspace/repositories/person/person.service.ts diff --git a/packages/twenty-server/src/workspace/messaging/repositories/workspace-member/workspace-member.module.ts b/packages/twenty-server/src/workspace/repositories/workspace-member/workspace-member.module.ts similarity index 65% rename from packages/twenty-server/src/workspace/messaging/repositories/workspace-member/workspace-member.module.ts rename to packages/twenty-server/src/workspace/repositories/workspace-member/workspace-member.module.ts index cb7538f62..1fbf387b7 100644 --- a/packages/twenty-server/src/workspace/messaging/repositories/workspace-member/workspace-member.module.ts +++ b/packages/twenty-server/src/workspace/repositories/workspace-member/workspace-member.module.ts @@ -1,9 +1,8 @@ import { Module } from '@nestjs/common'; -import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service'; +import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; -// TODO: Move outside of the messaging module @Module({ imports: [WorkspaceDataSourceModule], providers: [WorkspaceMemberService], diff --git a/packages/twenty-server/src/workspace/messaging/repositories/workspace-member/workspace-member.service.ts b/packages/twenty-server/src/workspace/repositories/workspace-member/workspace-member.service.ts similarity index 100% rename from packages/twenty-server/src/workspace/messaging/repositories/workspace-member/workspace-member.service.ts rename to packages/twenty-server/src/workspace/repositories/workspace-member/workspace-member.service.ts diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata.ts index 96c5dc2dd..cf9defbfc 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata.ts @@ -12,6 +12,7 @@ import { ObjectMetadata } from 'src/workspace/workspace-sync-metadata/decorators import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata'; import { CalendarChannelEventAssociationObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel-event-association.object-metadata'; import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata'; +import { IsNullable } from 'src/workspace/workspace-sync-metadata/decorators/is-nullable.decorator'; import { standardObjectIds } from 'src/workspace/workspace-sync-metadata/constants/standard-object-ids'; import { calendarEventStandardFieldIds } from 'src/workspace/workspace-sync-metadata/constants/standard-field-ids'; @@ -62,7 +63,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata { description: 'Start DateTime', icon: 'IconCalendarClock', }) - startsAt: string; + @IsNullable() + startsAt: string | null; @FieldMetadata({ standardId: calendarEventStandardFieldIds.endsAt, @@ -71,7 +73,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata { description: 'End DateTime', icon: 'IconCalendarClock', }) - endsAt: string; + @IsNullable() + endsAt: string | null; @FieldMetadata({ standardId: calendarEventStandardFieldIds.externalCreatedAt, @@ -80,7 +83,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata { description: 'Creation DateTime', icon: 'IconCalendarPlus', }) - externalCreatedAt: string; + @IsNullable() + externalCreatedAt: string | null; @FieldMetadata({ standardId: calendarEventStandardFieldIds.externalUpdatedAt, @@ -89,7 +93,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata { description: 'Update DateTime', icon: 'IconCalendarCog', }) - externalUpdatedAt: string; + @IsNullable() + externalUpdatedAt: string | null; @FieldMetadata({ standardId: calendarEventStandardFieldIds.description, diff --git a/packages/twenty-server/src/workspace/workspace.module.ts b/packages/twenty-server/src/workspace/workspace.module.ts index f6bbe9705..60ccb3c54 100644 --- a/packages/twenty-server/src/workspace/workspace.module.ts +++ b/packages/twenty-server/src/workspace/workspace.module.ts @@ -6,6 +6,7 @@ import { WorkspaceSchemaStorageModule } from 'src/workspace/workspace-schema-sto import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module'; import { ScalarsExplorerService } from 'src/workspace/services/scalars-explorer.service'; import { MessagingModule } from 'src/workspace/messaging/messaging.module'; +import { CalendarModule } from 'src/workspace/calendar/calendar.module'; import { WorkspaceFactory } from './workspace.factory'; @@ -21,6 +22,7 @@ import { WorkspaceResolverBuilderModule } from './workspace-resolver-builder/wor WorkspaceResolverBuilderModule, WorkspaceSchemaStorageModule, MessagingModule, + CalendarModule, ], providers: [WorkspaceFactory, ScalarsExplorerService], exports: [WorkspaceFactory],