4285 timebox create google calendar full sync (#4442)

* calendar module

* wip

* creating a folder for common files between calendar and messages

* wip

* wip

* wip

* wip

* update calendar search filter

* wip

* working on full sync service

* reorganizing folders

* adding repositories

* fix typo

* working on full-sync service

* Add calendarQueue to MessageQueue enum and update dependencies

* start transaction

* wip

* add save and update functions for event

* wip

* save events

* improving step by step

* add calendar scope

* fix nest modules imports

* renaming

* create calendar channel

* create job for google calendar full-sync

* call GoogleCalendarFullSyncJob after connected account creation

* ask for scope conditionnally

* fixes

* create channels conditionnally

* fix

* fixes

* fix FK bug

* filter out canceled events

* create save and update functions for calendarEventAttendee repository

* saving messageParticipants is working

* save calendarEventAttendees is working

* add calendarEvent cleaner

* calendar event cleaner is working

* working on updating attendees

* wip

* reintroducing google-gmail endpoint to ensure smooth deploy

* modify callbackURL

* modify front url

* changes to be able to merge

* put back feature flag

* fixes after PR comments

* add feature flag check

* remove unused modules

* separate delete connected account associated job data in two jobs

* fix error

* rename calendar_v3 as calendarV3

* Update packages/twenty-server/src/workspace/calendar-and-messaging/utils/valueStringForBatchRawQuery.util.ts

Co-authored-by: Jérémy M <jeremy.magrin@gmail.com>

* improve readability

* renaming to remove plural

* renaming to remove plural

* don't throw if no connected account is found

* use calendar queue

* modify usage of HttpService in fetch-by-batch

* modify valuesStringForBatchRawQuery to improve api and return flattened values

* fix auth module feature flag import

* fix getFlattenedValuesAndValuesStringForBatchRawQuery

---------

Co-authored-by: Jérémy M <jeremy.magrin@gmail.com>

packages/twenty-server/src/core/auth/auth.module.ts

@@ -14,12 +14,14 @@ import { UserModule } from 'src/core/user/user.module';
 import { WorkspaceManagerModule } from 'src/workspace/workspace-manager/workspace-manager.module';
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { GoogleAuthController } from 'src/core/auth/controllers/google-auth.controller';
-import { GoogleGmailAuthController } from 'src/core/auth/controllers/google-gmail-auth.controller';
+import { GoogleAPIsAuthController } from 'src/core/auth/controllers/google-apis-auth.controller';
 import { VerifyAuthController } from 'src/core/auth/controllers/verify-auth.controller';
 import { TokenService } from 'src/core/auth/services/token.service';
-import { GoogleGmailService } from 'src/core/auth/services/google-gmail.service';
+import { GoogleAPIsService } from 'src/core/auth/services/google-apis.service';
 import { UserWorkspaceModule } from 'src/core/user-workspace/user-workspace.module';
 import { SignUpService } from 'src/core/auth/services/sign-up.service';
+import { GoogleGmailAuthController } from 'src/core/auth/controllers/google-gmail-auth.controller';
+import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
 
 import { AuthResolver } from './auth.resolver';
 
@@ -45,12 +47,16 @@ const jwtModule = JwtModule.registerAsync({
     UserModule,
     WorkspaceManagerModule,
     TypeORMModule,
-    TypeOrmModule.forFeature([Workspace, User, RefreshToken], 'core'),
+    TypeOrmModule.forFeature(
+      [Workspace, User, RefreshToken, FeatureFlagEntity],
+      'core',
+    ),
     HttpModule,
     UserWorkspaceModule,
   ],
   controllers: [
     GoogleAuthController,
+    GoogleAPIsAuthController,
     GoogleGmailAuthController,
     VerifyAuthController,
   ],
@@ -60,7 +66,7 @@ const jwtModule = JwtModule.registerAsync({
     TokenService,
     JwtAuthStrategy,
     AuthResolver,
-    GoogleGmailService,
+    GoogleAPIsService,
   ],
   exports: [jwtModule, TokenService],
 })

packages/twenty-server/src/core/auth/controllers/google-apis-auth.controller.ts

@@ -0,0 +1,63 @@
+import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common';
+
+import { Response } from 'express';
+
+import { GoogleAPIsProviderEnabledGuard } from 'src/core/auth/guards/google-apis-provider-enabled.guard';
+import { GoogleAPIsOauthGuard } from 'src/core/auth/guards/google-apis-oauth.guard';
+import { GoogleAPIsRequest } from 'src/core/auth/strategies/google-apis.auth.strategy';
+import { GoogleAPIsService } from 'src/core/auth/services/google-apis.service';
+import { TokenService } from 'src/core/auth/services/token.service';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+
+@Controller('auth/google-apis')
+export class GoogleAPIsAuthController {
+  constructor(
+    private readonly googleAPIsService: GoogleAPIsService,
+    private readonly tokenService: TokenService,
+    private readonly environmentService: EnvironmentService,
+  ) {}
+
+  @Get()
+  @UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
+  async googleAuth() {
+    // As this method is protected by Google Auth guard, it will trigger Google SSO flow
+    return;
+  }
+
+  @Get('get-access-token')
+  @UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
+  async googleAuthGetAccessToken(
+    @Req() req: GoogleAPIsRequest,
+    @Res() res: Response,
+  ) {
+    const { user } = req;
+
+    const { email, accessToken, refreshToken, transientToken } = user;
+
+    const { workspaceMemberId, workspaceId } =
+      await this.tokenService.verifyTransientToken(transientToken);
+
+    const demoWorkspaceIds = this.environmentService.getDemoWorkspaceIds();
+
+    if (demoWorkspaceIds.includes(workspaceId)) {
+      throw new Error('Cannot connect Google account to demo workspace');
+    }
+
+    if (!workspaceId) {
+      throw new Error('Workspace not found');
+    }
+
+    await this.googleAPIsService.saveConnectedAccount({
+      handle: email,
+      workspaceMemberId: workspaceMemberId,
+      workspaceId: workspaceId,
+      provider: 'google',
+      accessToken,
+      refreshToken,
+    });
+
+    return res.redirect(
+      `${this.environmentService.getFrontBaseUrl()}/settings/accounts`,
+    );
+  }
+}

packages/twenty-server/src/core/auth/controllers/google-gmail-auth.controller.ts

@@ -2,32 +2,32 @@ import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common';
 
 import { Response } from 'express';
 
-import { GoogleGmailProviderEnabledGuard } from 'src/core/auth/guards/google-gmail-provider-enabled.guard';
-import { GoogleGmailOauthGuard } from 'src/core/auth/guards/google-gmail-oauth.guard';
-import { GoogleGmailRequest } from 'src/core/auth/strategies/google-gmail.auth.strategy';
-import { GoogleGmailService } from 'src/core/auth/services/google-gmail.service';
+import { GoogleAPIsOauthGuard } from 'src/core/auth/guards/google-apis-oauth.guard';
+import { GoogleAPIsProviderEnabledGuard } from 'src/core/auth/guards/google-apis-provider-enabled.guard';
+import { GoogleAPIsService } from 'src/core/auth/services/google-apis.service';
 import { TokenService } from 'src/core/auth/services/token.service';
+import { GoogleAPIsRequest } from 'src/core/auth/strategies/google-apis.auth.strategy';
 import { EnvironmentService } from 'src/integrations/environment/environment.service';
 
 @Controller('auth/google-gmail')
 export class GoogleGmailAuthController {
   constructor(
-    private readonly googleGmailService: GoogleGmailService,
+    private readonly googleGmailService: GoogleAPIsService,
     private readonly tokenService: TokenService,
     private readonly environmentService: EnvironmentService,
   ) {}
 
   @Get()
-  @UseGuards(GoogleGmailProviderEnabledGuard, GoogleGmailOauthGuard)
+  @UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
   async googleAuth() {
     // As this method is protected by Google Auth guard, it will trigger Google SSO flow
     return;
   }
 
   @Get('get-access-token')
-  @UseGuards(GoogleGmailProviderEnabledGuard, GoogleGmailOauthGuard)
+  @UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
   async googleAuthGetAccessToken(
-    @Req() req: GoogleGmailRequest,
+    @Req() req: GoogleAPIsRequest,
     @Res() res: Response,
   ) {
     const { user } = req;

packages/twenty-server/src/core/auth/guards/google-apis-oauth.guard.ts

@@ -2,7 +2,7 @@ import { ExecutionContext, Injectable } from '@nestjs/common';
 import { AuthGuard } from '@nestjs/passport';
 
 @Injectable()
-export class GoogleGmailOauthGuard extends AuthGuard('google-gmail') {
+export class GoogleAPIsOauthGuard extends AuthGuard('google-apis') {
   constructor() {
     super({
       prompt: 'select_account',

packages/twenty-server/src/core/auth/guards/google-apis-provider-enabled.guard.ts

@@ -0,0 +1,24 @@
+import { Injectable, CanActivate, NotFoundException } from '@nestjs/common';
+
+import { Observable } from 'rxjs';
+
+import { GoogleAPIsStrategy } from 'src/core/auth/strategies/google-apis.auth.strategy';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+
+@Injectable()
+export class GoogleAPIsProviderEnabledGuard implements CanActivate {
+  constructor(private readonly environmentService: EnvironmentService) {}
+
+  canActivate(): boolean | Promise<boolean> | Observable<boolean> {
+    if (
+      !this.environmentService.isMessagingProviderGmailEnabled() &&
+      !this.environmentService.isCalendarProviderGoogleEnabled()
+    ) {
+      throw new NotFoundException('Google apis auth is not enabled');
+    }
+
+    new GoogleAPIsStrategy(this.environmentService);
+
+    return true;
+  }
+}

packages/twenty-server/src/core/auth/guards/google-gmail-provider-enabled.guard.ts

@@ -1,21 +0,0 @@
-import { Injectable, CanActivate, NotFoundException } from '@nestjs/common';
-
-import { Observable } from 'rxjs';
-
-import { GoogleGmailStrategy } from 'src/core/auth/strategies/google-gmail.auth.strategy';
-import { EnvironmentService } from 'src/integrations/environment/environment.service';
-
-@Injectable()
-export class GoogleGmailProviderEnabledGuard implements CanActivate {
-  constructor(private readonly environmentService: EnvironmentService) {}
-
-  canActivate(): boolean | Promise<boolean> | Observable<boolean> {
-    if (!this.environmentService.isMessagingProviderGmailEnabled()) {
-      throw new NotFoundException('Gmail auth is not enabled');
-    }
-
-    new GoogleGmailStrategy(this.environmentService);
-
-    return true;
-  }
-}

packages/twenty-server/src/core/auth/services/google-apis.service.ts

@@ -1,6 +1,8 @@
 import { ConflictException, Inject, Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
 
 import { v4 } from 'uuid';
+import { Repository } from 'typeorm';
 
 import { DataSourceService } from 'src/metadata/data-source/data-source.service';
 import { TypeORMService } from 'src/database/typeorm/typeorm.service';
@@ -11,14 +13,28 @@ import {
   GmailFullSyncJob,
   GmailFullSyncJobData,
 } from 'src/workspace/messaging/jobs/gmail-full-sync.job';
+import {
+  GoogleCalendarFullSyncJob,
+  GoogleCalendarFullSyncJobData,
+} from 'src/workspace/calendar/jobs/google-calendar-full-sync.job';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+import {
+  FeatureFlagEntity,
+  FeatureFlagKeys,
+} from 'src/core/feature-flag/feature-flag.entity';
 
 @Injectable()
-export class GoogleGmailService {
+export class GoogleAPIsService {
   constructor(
     private readonly dataSourceService: DataSourceService,
     private readonly typeORMService: TypeORMService,
     @Inject(MessageQueue.messagingQueue)
     private readonly messageQueueService: MessageQueueService,
+    @Inject(MessageQueue.calendarQueue)
+    private readonly calendarQueueService: MessageQueueService,
+    private readonly environmentService: EnvironmentService,
+    @InjectRepository(FeatureFlagEntity, 'core')
+    private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
   ) {}
 
   providerName = 'google';
@@ -53,6 +69,12 @@ export class GoogleGmailService {
 
     const connectedAccountId = v4();
 
+    const IsCalendarEnabled = await this.featureFlagRepository.findOneBy({
+      workspaceId,
+      key: FeatureFlagKeys.IsCalendarEnabled,
+      value: true,
+    });
+
     await workspaceDataSource?.transaction(async (manager) => {
       await manager.query(
         `INSERT INTO ${dataSourceMetadata.schema}."connectedAccount" ("id", "handle", "provider", "accessToken", "refreshToken", "accountOwnerId") VALUES ($1, $2, $3, $4, $5, $6)`,
@@ -66,22 +88,52 @@ export class GoogleGmailService {
         ],
       );
 
-      await manager.query(
-        `INSERT INTO ${dataSourceMetadata.schema}."messageChannel" ("visibility", "handle", "connectedAccountId", "type") VALUES ($1, $2, $3, $4)`,
-        ['share_everything', handle, connectedAccountId, 'email'],
-      );
+      if (this.environmentService.isMessagingProviderGmailEnabled()) {
+        await manager.query(
+          `INSERT INTO ${dataSourceMetadata.schema}."messageChannel" ("visibility", "handle", "connectedAccountId", "type") VALUES ($1, $2, $3, $4)`,
+          ['share_everything', handle, connectedAccountId, 'email'],
+        );
+      }
+
+      if (
+        this.environmentService.isCalendarProviderGoogleEnabled() &&
+        IsCalendarEnabled
+      ) {
+        await manager.query(
+          `INSERT INTO ${dataSourceMetadata.schema}."calendarChannel" ("visibility", "handle", "connectedAccountId") VALUES ($1, $2, $3)`,
+          ['SHARE_EVERYTHING', handle, connectedAccountId],
+        );
+      }
     });
 
-    await this.messageQueueService.add<GmailFullSyncJobData>(
-      GmailFullSyncJob.name,
-      {
-        workspaceId,
-        connectedAccountId,
-      },
-      {
-        retryLimit: 2,
-      },
-    );
+    if (this.environmentService.isMessagingProviderGmailEnabled()) {
+      await this.messageQueueService.add<GmailFullSyncJobData>(
+        GmailFullSyncJob.name,
+        {
+          workspaceId,
+          connectedAccountId,
+        },
+        {
+          retryLimit: 2,
+        },
+      );
+    }
+
+    if (
+      this.environmentService.isCalendarProviderGoogleEnabled() &&
+      IsCalendarEnabled
+    ) {
+      await this.calendarQueueService.add<GoogleCalendarFullSyncJobData>(
+        GoogleCalendarFullSyncJob.name,
+        {
+          workspaceId,
+          connectedAccountId,
+        },
+        {
+          retryLimit: 2,
+        },
+      );
+    }
 
     return;
   }

packages/twenty-server/src/core/auth/strategies/google-apis.auth.strategy.ts

@@ -6,7 +6,7 @@ import { Request } from 'express';
 
 import { EnvironmentService } from 'src/integrations/environment/environment.service';
 
-export type GoogleGmailRequest = Request & {
+export type GoogleAPIsRequest = Request & {
   user: {
     firstName?: string | null;
     lastName?: string | null;
@@ -20,20 +20,28 @@ export type GoogleGmailRequest = Request & {
 };
 
 @Injectable()
-export class GoogleGmailStrategy extends PassportStrategy(
+export class GoogleAPIsStrategy extends PassportStrategy(
   Strategy,
-  'google-gmail',
+  'google-apis',
 ) {
   constructor(environmentService: EnvironmentService) {
+    const scope = ['email', 'profile'];
+
+    if (environmentService.isMessagingProviderGmailEnabled()) {
+      scope.push('https://www.googleapis.com/auth/gmail.readonly');
+    }
+
+    if (environmentService.isCalendarProviderGoogleEnabled()) {
+      scope.push('https://www.googleapis.com/auth/calendar');
+    }
+
     super({
       clientID: environmentService.getAuthGoogleClientId(),
       clientSecret: environmentService.getAuthGoogleClientSecret(),
-      callbackURL: environmentService.getMessagingProviderGmailCallbackUrl(),
-      scope: [
-        'email',
-        'profile',
-        'https://www.googleapis.com/auth/gmail.readonly',
-      ],
+      callbackURL: environmentService.isCalendarProviderGoogleEnabled()
+        ? environmentService.getAuthGoogleAPIsCallbackUrl()
+        : environmentService.getMessagingProviderGmailCallbackUrl(),
+      scope,
       passReqToCallback: true,
     });
   }
@@ -52,7 +60,7 @@ export class GoogleGmailStrategy extends PassportStrategy(
   }
 
   async validate(
-    request: GoogleGmailRequest,
+    request: GoogleAPIsRequest,
     accessToken: string,
     refreshToken: string,
     profile: any,
@@ -65,7 +73,7 @@ export class GoogleGmailStrategy extends PassportStrategy(
         ? JSON.parse(request.query.state)
         : undefined;
 
-    const user: GoogleGmailRequest['user'] = {
+    const user: GoogleAPIsRequest['user'] = {
       email: emails[0].value,
       firstName: name.givenName,
       lastName: name.familyName,