o365 calendar sync (#8044)

Implemented:

* Account Connect
* Calendar sync via delta ids then requesting single events


I think I would split the messaging part into a second pr - that's a
step more complex then the calendar :)

---------

Co-authored-by: bosiraphael <raphael.bosi@gmail.com>
This commit is contained in:
brendanlaschke
2024-11-07 18:13:22 +01:00
committed by GitHub
parent 83f3963bfb
commit f9c076df31
50 changed files with 1417 additions and 118 deletions

View File

@ -8,11 +8,13 @@ import { AppToken } from 'src/engine/core-modules/app-token/app-token.entity';
import { AppTokenService } from 'src/engine/core-modules/app-token/services/app-token.service';
import { GoogleAPIsAuthController } from 'src/engine/core-modules/auth/controllers/google-apis-auth.controller';
import { GoogleAuthController } from 'src/engine/core-modules/auth/controllers/google-auth.controller';
import { MicrosoftAPIsAuthController } from 'src/engine/core-modules/auth/controllers/microsoft-apis-auth.controller';
import { MicrosoftAuthController } from 'src/engine/core-modules/auth/controllers/microsoft-auth.controller';
import { SSOAuthController } from 'src/engine/core-modules/auth/controllers/sso-auth.controller';
import { VerifyAuthController } from 'src/engine/core-modules/auth/controllers/verify-auth.controller';
import { ApiKeyService } from 'src/engine/core-modules/auth/services/api-key.service';
import { GoogleAPIsService } from 'src/engine/core-modules/auth/services/google-apis.service';
import { MicrosoftAPIsService } from 'src/engine/core-modules/auth/services/microsoft-apis.service';
import { OAuthService } from 'src/engine/core-modules/auth/services/oauth.service';
import { ResetPasswordService } from 'src/engine/core-modules/auth/services/reset-password.service';
import { SignInUpService } from 'src/engine/core-modules/auth/services/sign-in-up.service';
@ -80,6 +82,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy';
GoogleAuthController,
MicrosoftAuthController,
GoogleAPIsAuthController,
MicrosoftAPIsAuthController,
VerifyAuthController,
SSOAuthController,
],
@ -90,6 +93,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy';
SamlAuthStrategy,
AuthResolver,
GoogleAPIsService,
MicrosoftAPIsService,
AppTokenService,
AccessTokenService,
LoginTokenService,

View File

@ -0,0 +1,105 @@
import {
Controller,
Get,
Req,
Res,
UseFilters,
UseGuards,
} from '@nestjs/common';
import { Response } from 'express';
import {
AuthException,
AuthExceptionCode,
} from 'src/engine/core-modules/auth/auth.exception';
import { AuthRestApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-rest-api-exception.filter';
import { MicrosoftAPIsOauthExchangeCodeForTokenGuard } from 'src/engine/core-modules/auth/guards/microsoft-apis-oauth-exchange-code-for-token.guard';
import { MicrosoftAPIsOauthRequestCodeGuard } from 'src/engine/core-modules/auth/guards/mircosoft-apis-oauth-request-code.guard';
import { MicrosoftAPIsService } from 'src/engine/core-modules/auth/services/microsoft-apis.service';
import { TransientTokenService } from 'src/engine/core-modules/auth/token/services/transient-token.service';
import { MicrosoftAPIsRequest } from 'src/engine/core-modules/auth/types/microsoft-api-request.type';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service';
@Controller('auth/microsoft-apis')
@UseFilters(AuthRestApiExceptionFilter)
export class MicrosoftAPIsAuthController {
constructor(
private readonly microsoftAPIsService: MicrosoftAPIsService,
private readonly transientTokenService: TransientTokenService,
private readonly environmentService: EnvironmentService,
private readonly onboardingService: OnboardingService,
) {}
@Get()
@UseGuards(MicrosoftAPIsOauthRequestCodeGuard)
async MicrosoftAuth() {
// As this method is protected by Microsoft Auth guard, it will trigger Microsoft SSO flow
return;
}
@Get('get-access-token')
@UseGuards(MicrosoftAPIsOauthExchangeCodeForTokenGuard)
async MicrosoftAuthGetAccessToken(
@Req() req: MicrosoftAPIsRequest,
@Res() res: Response,
) {
const { user } = req;
const {
emails,
accessToken,
refreshToken,
transientToken,
redirectLocation,
calendarVisibility,
messageVisibility,
} = user;
const { workspaceMemberId, userId, workspaceId } =
await this.transientTokenService.verifyTransientToken(transientToken);
const demoWorkspaceIds = this.environmentService.get('DEMO_WORKSPACE_IDS');
if (demoWorkspaceIds.includes(workspaceId)) {
throw new AuthException(
'Cannot connect Microsoft account to demo workspace',
AuthExceptionCode.FORBIDDEN_EXCEPTION,
);
}
if (!workspaceId) {
throw new AuthException(
'Workspace not found',
AuthExceptionCode.WORKSPACE_NOT_FOUND,
);
}
const handle = emails[0].value;
await this.microsoftAPIsService.refreshMicrosoftRefreshToken({
handle,
workspaceMemberId: workspaceMemberId,
workspaceId: workspaceId,
accessToken,
refreshToken,
calendarVisibility,
messageVisibility,
});
if (userId) {
await this.onboardingService.setOnboardingConnectAccountPending({
userId,
workspaceId,
value: false,
});
}
return res.redirect(
`${this.environmentService.get('FRONT_BASE_URL')}${
redirectLocation || '/settings/accounts'
}`,
);
}
}

View File

@ -0,0 +1,31 @@
import { ExecutionContext, Injectable } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
import { MicrosoftAPIsOauthExchangeCodeForTokenStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-exchange-code-for-token.auth.strategy';
import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
@Injectable()
export class MicrosoftAPIsOauthExchangeCodeForTokenGuard extends AuthGuard(
'microsoft-apis',
) {
constructor(private readonly environmentService: EnvironmentService) {
super();
}
async canActivate(context: ExecutionContext) {
const request = context.switchToHttp().getRequest();
const state = JSON.parse(request.query.state);
new MicrosoftAPIsOauthExchangeCodeForTokenStrategy(this.environmentService);
setRequestExtraParams(request, {
transientToken: state.transientToken,
redirectLocation: state.redirectLocation,
calendarVisibility: state.calendarVisibility,
messageVisibility: state.messageVisibility,
});
return (await super.canActivate(context)) as boolean;
}
}

View File

@ -0,0 +1,62 @@
import { ExecutionContext, Injectable } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
import {
AuthException,
AuthExceptionCode,
} from 'src/engine/core-modules/auth/auth.exception';
import { MicrosoftAPIsOauthRequestCodeStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-request-code.auth.strategy';
import { TransientTokenService } from 'src/engine/core-modules/auth/token/services/transient-token.service';
import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
@Injectable()
export class MicrosoftAPIsOauthRequestCodeGuard extends AuthGuard(
'microsoft-apis',
) {
constructor(
private readonly environmentService: EnvironmentService,
private readonly featureFlagService: FeatureFlagService,
private readonly transientTokenService: TransientTokenService,
) {
super({
prompt: 'select_account',
});
}
async canActivate(context: ExecutionContext) {
const request = context.switchToHttp().getRequest();
const { workspaceId } =
await this.transientTokenService.verifyTransientToken(
request.query.transientToken,
);
const isMicrosoftSyncEnabled =
await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IsMicrosoftSyncEnabled,
workspaceId,
);
if (!isMicrosoftSyncEnabled) {
throw new AuthException(
'Microsoft sync is not enabled',
AuthExceptionCode.FORBIDDEN_EXCEPTION,
);
}
new MicrosoftAPIsOauthRequestCodeStrategy(this.environmentService);
setRequestExtraParams(request, {
transientToken: request.query.transientToken,
redirectLocation: request.query.redirectLocation,
calendarVisibility: request.query.calendarVisibility,
messageVisibility: request.query.messageVisibility,
loginHint: request.query.loginHint,
});
const activate = (await super.canActivate(context)) as boolean;
return activate;
}
}

View File

@ -3,14 +3,17 @@ import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { getGoogleApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-google-apis-oauth-scopes';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import {
CalendarEventListFetchJob,
CalendarEventsImportJobData,
CalendarEventListFetchJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import {
CalendarChannelVisibility,
@ -33,9 +36,6 @@ import {
MessagingMessageListFetchJobData,
} from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { getGoogleApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-google-apis-oauth-scopes';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
@Injectable()
export class GoogleAPIsService {
@ -222,7 +222,7 @@ export class GoogleAPIsService {
});
for (const calendarChannel of calendarChannels) {
await this.calendarQueueService.add<CalendarEventsImportJobData>(
await this.calendarQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,

View File

@ -0,0 +1,212 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { getMicrosoftApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-microsoft-apis-oauth-scopes';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import {
CalendarEventListFetchJob,
CalendarEventListFetchJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import {
CalendarChannelVisibility,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
import {
ConnectedAccountProvider,
ConnectedAccountWorkspaceEntity,
} from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import {
MessageChannelSyncStage,
MessageChannelSyncStatus,
MessageChannelType,
MessageChannelVisibility,
MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@Injectable()
export class MicrosoftAPIsService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@InjectMessageQueue(MessageQueue.calendarQueue)
private readonly calendarQueueService: MessageQueueService,
private readonly accountsToReconnectService: AccountsToReconnectService,
) {}
async refreshMicrosoftRefreshToken(input: {
handle: string;
workspaceMemberId: string;
workspaceId: string;
accessToken: string;
refreshToken: string;
calendarVisibility: CalendarChannelVisibility | undefined;
messageVisibility: MessageChannelVisibility | undefined;
}) {
const {
handle,
workspaceId,
workspaceMemberId,
calendarVisibility,
messageVisibility,
} = input;
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const connectedAccount = await connectedAccountRepository.findOne({
where: { handle, accountOwnerId: workspaceMemberId },
});
const existingAccountId = connectedAccount?.id;
const newOrExistingConnectedAccountId = existingAccountId ?? v4();
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace(workspaceId);
const scopes = getMicrosoftApisOauthScopes();
await workspaceDataSource.transaction(async (manager: EntityManager) => {
if (!existingAccountId) {
await connectedAccountRepository.save(
{
id: newOrExistingConnectedAccountId,
handle,
provider: ConnectedAccountProvider.MICROSOFT,
accessToken: input.accessToken,
refreshToken: input.refreshToken,
accountOwnerId: workspaceMemberId,
scopes,
},
{},
manager,
);
// TODO: Modify this when the email sync is implemented
await messageChannelRepository.save(
{
id: v4(),
connectedAccountId: newOrExistingConnectedAccountId,
type: MessageChannelType.EMAIL,
handle,
visibility:
messageVisibility || MessageChannelVisibility.SHARE_EVERYTHING,
syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
syncStage: MessageChannelSyncStage.FAILED,
},
{},
manager,
);
await calendarChannelRepository.save(
{
id: v4(),
connectedAccountId: newOrExistingConnectedAccountId,
handle,
visibility:
calendarVisibility || CalendarChannelVisibility.SHARE_EVERYTHING,
},
{},
manager,
);
} else {
await connectedAccountRepository.update(
{
id: newOrExistingConnectedAccountId,
},
{
accessToken: input.accessToken,
refreshToken: input.refreshToken,
scopes,
},
manager,
);
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);
const workspaceMember = await workspaceMemberRepository.findOneOrFail({
where: { id: workspaceMemberId },
});
const userId = workspaceMember.userId;
await this.accountsToReconnectService.removeAccountToReconnect(
userId,
workspaceId,
newOrExistingConnectedAccountId,
);
// TODO: Modify this when the email sync is implemented
await messageChannelRepository.update(
{
connectedAccountId: newOrExistingConnectedAccountId,
},
{
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
syncCursor: '',
syncStageStartedAt: null,
},
manager,
);
}
});
// TODO: Uncomment this when the email sync is implemented
// const messageChannels = await messageChannelRepository.find({
// where: {
// connectedAccountId: newOrExistingConnectedAccountId,
// },
// });
// for (const messageChannel of messageChannels) {
// await this.messageQueueService.add<MessagingMessageListFetchJobData>(
// MessagingMessageListFetchJob.name,
// {
// workspaceId,
// messageChannelId: messageChannel.id,
// },
// );
// }
const calendarChannels = await calendarChannelRepository.find({
where: {
connectedAccountId: newOrExistingConnectedAccountId,
},
});
for (const calendarChannel of calendarChannels) {
await this.calendarQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,
workspaceId,
},
);
}
}
}

View File

@ -0,0 +1,31 @@
import { Injectable } from '@nestjs/common';
import { PassportStrategy } from '@nestjs/passport';
import { Strategy } from 'passport-microsoft';
import { getMicrosoftApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-microsoft-apis-oauth-scopes';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
export type MicrosoftAPIScopeConfig = {
isCalendarEnabled?: boolean;
isMessagingAliasFetchingEnabled?: boolean;
};
@Injectable()
export class MicrosoftAPIsOauthCommonStrategy extends PassportStrategy(
Strategy,
'microsoft-apis',
) {
constructor(environmentService: EnvironmentService) {
const scopes = getMicrosoftApisOauthScopes();
super({
clientID: environmentService.get('AUTH_MICROSOFT_CLIENT_ID'),
clientSecret: environmentService.get('AUTH_MICROSOFT_CLIENT_SECRET'),
tenant: environmentService.get('AUTH_MICROSOFT_TENANT_ID'),
callbackURL: environmentService.get('AUTH_MICROSOFT_APIS_CALLBACK_URL'),
scope: scopes,
passReqToCallback: true,
});
}
}

View File

@ -0,0 +1,48 @@
import { Injectable } from '@nestjs/common';
import { VerifyCallback } from 'passport-google-oauth20';
import { MicrosoftAPIsOauthCommonStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-common.auth.strategy';
import { MicrosoftAPIsRequest } from 'src/engine/core-modules/auth/types/microsoft-api-request.type';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
export type MicrosoftAPIScopeConfig = {
isCalendarEnabled?: boolean;
};
@Injectable()
export class MicrosoftAPIsOauthExchangeCodeForTokenStrategy extends MicrosoftAPIsOauthCommonStrategy {
constructor(environmentService: EnvironmentService) {
super(environmentService);
}
async validate(
request: MicrosoftAPIsRequest,
accessToken: string,
refreshToken: string,
profile: any,
done: VerifyCallback,
): Promise<void> {
const { name, emails, photos } = profile;
const state =
typeof request.query.state === 'string'
? JSON.parse(request.query.state)
: undefined;
const user: MicrosoftAPIsRequest['user'] = {
emails,
firstName: name.givenName,
lastName: name.familyName,
picture: photos?.[0]?.value,
accessToken,
refreshToken,
transientToken: state.transientToken,
redirectLocation: state.redirectLocation,
calendarVisibility: state.calendarVisibility,
messageVisibility: state.messageVisibility,
};
done(null, user);
}
}

View File

@ -0,0 +1,28 @@
import { Injectable } from '@nestjs/common';
import { MicrosoftAPIsOauthCommonStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-common.auth.strategy';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
@Injectable()
export class MicrosoftAPIsOauthRequestCodeStrategy extends MicrosoftAPIsOauthCommonStrategy {
constructor(environmentService: EnvironmentService) {
super(environmentService);
}
authenticate(req: any, options: any) {
options = {
...options,
accessType: 'offline',
prompt: 'consent',
loginHint: req.params.loginHint,
state: JSON.stringify({
transientToken: req.params.transientToken,
redirectLocation: req.params.redirectLocation,
calendarVisibility: req.params.calendarVisibility,
messageVisibility: req.params.messageVisibility,
}),
};
return super.authenticate(req, options);
}
}

View File

@ -0,0 +1,23 @@
import { Request } from 'express';
import { CalendarChannelVisibility } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { MessageChannelVisibility } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
export type MicrosoftAPIsRequest = Omit<
Request,
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;
lastName?: string | null;
emails: { value: string }[];
picture: string | null;
workspaceInviteHash?: string;
accessToken: string;
refreshToken: string;
transientToken: string;
redirectLocation?: string;
calendarVisibility?: CalendarChannelVisibility;
messageVisibility?: MessageChannelVisibility;
};
};

View File

@ -0,0 +1,12 @@
export const getMicrosoftApisOauthScopes = () => {
const scopes = [
'openid',
'email',
'profile',
'offline_access',
'Mail.Read',
'Calendars.Read',
];
return scopes;
};

View File

@ -201,6 +201,10 @@ export class EnvironmentVariables {
@ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED)
AUTH_MICROSOFT_CALLBACK_URL: string;
@IsUrl({ require_tld: false })
@ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED)
AUTH_MICROSOFT_APIS_CALLBACK_URL: string;
@CastToBoolean()
@IsOptional()
@IsBoolean()

View File

@ -14,5 +14,6 @@ export enum FeatureFlagKey {
IsGmailSendEmailScopeEnabled = 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED',
IsAnalyticsV2Enabled = 'IS_ANALYTICS_V2_ENABLED',
IsUniqueIndexesEnabled = 'IS_UNIQUE_INDEXES_ENABLED',
IsMicrosoftSyncEnabled = 'IS_MICROSOFT_SYNC_ENABLED',
IsAdvancedFiltersEnabled = 'IS_ADVANCED_FILTERS_ENABLED',
}

View File

@ -35,7 +35,6 @@ import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorat
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { isDefined } from 'src/utils/is-defined';
const getHMACKey = (email?: string, key?: string | null) => {
if (!email || !key) return null;