admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

o365 calendar sync (#8044)

Browse Source 
Implemented:

* Account Connect
* Calendar sync via delta ids then requesting single events


I think I would split the messaging part into a second pr - that's a
step more complex then the calendar :)

---------

Co-authored-by: bosiraphael <raphael.bosi@gmail.com>
This commit is contained in:
brendanlaschke
2024-11-07 18:13:22 +01:00
committed by GitHub
parent 83f3963bfb
commit f9c076df31
50 changed files with 1417 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/twenty-server/.env.example
View File

@ -29,6 +29,7 @@ ACCESS_TOKEN_SECRET=replace_me_with_a_random_string_access
# AUTH_MICROSOFT_TENANT_ID=replace_me_with_azure_tenant_id
# AUTH_MICROSOFT_CLIENT_SECRET=replace_me_with_azure_client_secret
# AUTH_MICROSOFT_CALLBACK_URL=http://localhost:3000/auth/microsoft/redirect
# AUTH_MICROSOFT_APIS_CALLBACK_URL=http://localhost:3000/auth/microsoft-apis/get-access-token
# AUTH_GOOGLE_ENABLED=false
# AUTH_GOOGLE_CLIENT_ID=replace_me_with_google_client_id
# AUTH_GOOGLE_CLIENT_SECRET=replace_me_with_google_client_secret

5
packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts
View File

@ -75,6 +75,11 @@ export const seedFeatureFlags = async (
workspaceId: workspaceId,
value: false,
},
{
key: FeatureFlagKey.IsMicrosoftSyncEnabled,
workspaceId: workspaceId,
value: true,
},
{
key: FeatureFlagKey.IsAdvancedFiltersEnabled,
workspaceId: workspaceId,

4
packages/twenty-server/src/engine/core-modules/auth/auth.module.ts
View File

@ -8,11 +8,13 @@ import { AppToken } from 'src/engine/core-modules/app-token/app-token.entity';
import { AppTokenService } from 'src/engine/core-modules/app-token/services/app-token.service';
import { GoogleAPIsAuthController } from 'src/engine/core-modules/auth/controllers/google-apis-auth.controller';
import { GoogleAuthController } from 'src/engine/core-modules/auth/controllers/google-auth.controller';
import { MicrosoftAPIsAuthController } from 'src/engine/core-modules/auth/controllers/microsoft-apis-auth.controller';
import { MicrosoftAuthController } from 'src/engine/core-modules/auth/controllers/microsoft-auth.controller';
import { SSOAuthController } from 'src/engine/core-modules/auth/controllers/sso-auth.controller';
import { VerifyAuthController } from 'src/engine/core-modules/auth/controllers/verify-auth.controller';
import { ApiKeyService } from 'src/engine/core-modules/auth/services/api-key.service';
import { GoogleAPIsService } from 'src/engine/core-modules/auth/services/google-apis.service';
import { MicrosoftAPIsService } from 'src/engine/core-modules/auth/services/microsoft-apis.service';
import { OAuthService } from 'src/engine/core-modules/auth/services/oauth.service';
import { ResetPasswordService } from 'src/engine/core-modules/auth/services/reset-password.service';
import { SignInUpService } from 'src/engine/core-modules/auth/services/sign-in-up.service';
@ -80,6 +82,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy';
GoogleAuthController,
MicrosoftAuthController,
GoogleAPIsAuthController,
MicrosoftAPIsAuthController,
VerifyAuthController,
SSOAuthController,
],
@ -90,6 +93,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy';
SamlAuthStrategy,
AuthResolver,
GoogleAPIsService,
MicrosoftAPIsService,
AppTokenService,
AccessTokenService,
LoginTokenService,

105
packages/twenty-server/src/engine/core-modules/auth/controllers/microsoft-apis-auth.controller.ts Normal file
View File

@ -0,0 +1,105 @@
import {
Controller,
Get,
Req,
Res,
UseFilters,
UseGuards,
} from '@nestjs/common';
import { Response } from 'express';
import {
AuthException,
AuthExceptionCode,
} from 'src/engine/core-modules/auth/auth.exception';
import { AuthRestApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-rest-api-exception.filter';
import { MicrosoftAPIsOauthExchangeCodeForTokenGuard } from 'src/engine/core-modules/auth/guards/microsoft-apis-oauth-exchange-code-for-token.guard';
import { MicrosoftAPIsOauthRequestCodeGuard } from 'src/engine/core-modules/auth/guards/mircosoft-apis-oauth-request-code.guard';
import { MicrosoftAPIsService } from 'src/engine/core-modules/auth/services/microsoft-apis.service';
import { TransientTokenService } from 'src/engine/core-modules/auth/token/services/transient-token.service';
import { MicrosoftAPIsRequest } from 'src/engine/core-modules/auth/types/microsoft-api-request.type';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service';
@Controller('auth/microsoft-apis')
@UseFilters(AuthRestApiExceptionFilter)
export class MicrosoftAPIsAuthController {
constructor(
private readonly microsoftAPIsService: MicrosoftAPIsService,
private readonly transientTokenService: TransientTokenService,
private readonly environmentService: EnvironmentService,
private readonly onboardingService: OnboardingService,
) {}
@Get()
@UseGuards(MicrosoftAPIsOauthRequestCodeGuard)
async MicrosoftAuth() {
// As this method is protected by Microsoft Auth guard, it will trigger Microsoft SSO flow
return;
}
@Get('get-access-token')
@UseGuards(MicrosoftAPIsOauthExchangeCodeForTokenGuard)
async MicrosoftAuthGetAccessToken(
@Req() req: MicrosoftAPIsRequest,
@Res() res: Response,
) {
const { user } = req;
const {
emails,
accessToken,
refreshToken,
transientToken,
redirectLocation,
calendarVisibility,
messageVisibility,
} = user;
const { workspaceMemberId, userId, workspaceId } =
await this.transientTokenService.verifyTransientToken(transientToken);
const demoWorkspaceIds = this.environmentService.get('DEMO_WORKSPACE_IDS');
if (demoWorkspaceIds.includes(workspaceId)) {
throw new AuthException(
'Cannot connect Microsoft account to demo workspace',
AuthExceptionCode.FORBIDDEN_EXCEPTION,
);
}
if (!workspaceId) {
throw new AuthException(
'Workspace not found',
AuthExceptionCode.WORKSPACE_NOT_FOUND,
);
}
const handle = emails[0].value;
await this.microsoftAPIsService.refreshMicrosoftRefreshToken({
handle,
workspaceMemberId: workspaceMemberId,
workspaceId: workspaceId,
accessToken,
refreshToken,
calendarVisibility,
messageVisibility,
});
if (userId) {
await this.onboardingService.setOnboardingConnectAccountPending({
userId,
workspaceId,
value: false,
});
}
return res.redirect(
`${this.environmentService.get('FRONT_BASE_URL')}${
redirectLocation || '/settings/accounts'
}`,
);
}
}

31
packages/twenty-server/src/engine/core-modules/auth/guards/microsoft-apis-oauth-exchange-code-for-token.guard.ts Normal file
View File

@ -0,0 +1,31 @@
import { ExecutionContext, Injectable } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
import { MicrosoftAPIsOauthExchangeCodeForTokenStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-exchange-code-for-token.auth.strategy';
import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
@Injectable()
export class MicrosoftAPIsOauthExchangeCodeForTokenGuard extends AuthGuard(
'microsoft-apis',
) {
constructor(private readonly environmentService: EnvironmentService) {
super();
}
async canActivate(context: ExecutionContext) {
const request = context.switchToHttp().getRequest();
const state = JSON.parse(request.query.state);
new MicrosoftAPIsOauthExchangeCodeForTokenStrategy(this.environmentService);
setRequestExtraParams(request, {
transientToken: state.transientToken,
redirectLocation: state.redirectLocation,
calendarVisibility: state.calendarVisibility,
messageVisibility: state.messageVisibility,
});
return (await super.canActivate(context)) as boolean;
}
}

62
packages/twenty-server/src/engine/core-modules/auth/guards/mircosoft-apis-oauth-request-code.guard.ts Normal file
View File

@ -0,0 +1,62 @@
import { ExecutionContext, Injectable } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
import {
AuthException,
AuthExceptionCode,
} from 'src/engine/core-modules/auth/auth.exception';
import { MicrosoftAPIsOauthRequestCodeStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-request-code.auth.strategy';
import { TransientTokenService } from 'src/engine/core-modules/auth/token/services/transient-token.service';
import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
@Injectable()
export class MicrosoftAPIsOauthRequestCodeGuard extends AuthGuard(
'microsoft-apis',
) {
constructor(
private readonly environmentService: EnvironmentService,
private readonly featureFlagService: FeatureFlagService,
private readonly transientTokenService: TransientTokenService,
) {
super({
prompt: 'select_account',
});
}
async canActivate(context: ExecutionContext) {
const request = context.switchToHttp().getRequest();
const { workspaceId } =
await this.transientTokenService.verifyTransientToken(
request.query.transientToken,
);
const isMicrosoftSyncEnabled =
await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IsMicrosoftSyncEnabled,
workspaceId,
);
if (!isMicrosoftSyncEnabled) {
throw new AuthException(
'Microsoft sync is not enabled',
AuthExceptionCode.FORBIDDEN_EXCEPTION,
);
}
new MicrosoftAPIsOauthRequestCodeStrategy(this.environmentService);
setRequestExtraParams(request, {
transientToken: request.query.transientToken,
redirectLocation: request.query.redirectLocation,
calendarVisibility: request.query.calendarVisibility,
messageVisibility: request.query.messageVisibility,
loginHint: request.query.loginHint,
});
const activate = (await super.canActivate(context)) as boolean;
return activate;
}
}

10
packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts
View File

@ -3,14 +3,17 @@ import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { getGoogleApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-google-apis-oauth-scopes';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import {
CalendarEventListFetchJob,
CalendarEventsImportJobData,
CalendarEventListFetchJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import {
CalendarChannelVisibility,
@ -33,9 +36,6 @@ import {
MessagingMessageListFetchJobData,
} from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { getGoogleApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-google-apis-oauth-scopes';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
@Injectable()
export class GoogleAPIsService {
@ -222,7 +222,7 @@ export class GoogleAPIsService {
});
for (const calendarChannel of calendarChannels) {
await this.calendarQueueService.add<CalendarEventsImportJobData>(
await this.calendarQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,

212
packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts Normal file
View File

@ -0,0 +1,212 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { getMicrosoftApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-microsoft-apis-oauth-scopes';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import {
CalendarEventListFetchJob,
CalendarEventListFetchJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import {
CalendarChannelVisibility,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
import {
ConnectedAccountProvider,
ConnectedAccountWorkspaceEntity,
} from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import {
MessageChannelSyncStage,
MessageChannelSyncStatus,
MessageChannelType,
MessageChannelVisibility,
MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@Injectable()
export class MicrosoftAPIsService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@InjectMessageQueue(MessageQueue.calendarQueue)
private readonly calendarQueueService: MessageQueueService,
private readonly accountsToReconnectService: AccountsToReconnectService,
) {}
async refreshMicrosoftRefreshToken(input: {
handle: string;
workspaceMemberId: string;
workspaceId: string;
accessToken: string;
refreshToken: string;
calendarVisibility: CalendarChannelVisibility | undefined;
messageVisibility: MessageChannelVisibility | undefined;
}) {
const {
handle,
workspaceId,
workspaceMemberId,
calendarVisibility,
messageVisibility,
} = input;
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const connectedAccount = await connectedAccountRepository.findOne({
where: { handle, accountOwnerId: workspaceMemberId },
});
const existingAccountId = connectedAccount?.id;
const newOrExistingConnectedAccountId = existingAccountId ?? v4();
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace(workspaceId);
const scopes = getMicrosoftApisOauthScopes();
await workspaceDataSource.transaction(async (manager: EntityManager) => {
if (!existingAccountId) {
await connectedAccountRepository.save(
{
id: newOrExistingConnectedAccountId,
handle,
provider: ConnectedAccountProvider.MICROSOFT,
accessToken: input.accessToken,
refreshToken: input.refreshToken,
accountOwnerId: workspaceMemberId,
scopes,
},
{},
manager,
);
// TODO: Modify this when the email sync is implemented
await messageChannelRepository.save(
{
id: v4(),
connectedAccountId: newOrExistingConnectedAccountId,
type: MessageChannelType.EMAIL,
handle,
visibility:
messageVisibility || MessageChannelVisibility.SHARE_EVERYTHING,
syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
syncStage: MessageChannelSyncStage.FAILED,
},
{},
manager,
);
await calendarChannelRepository.save(
{
id: v4(),
connectedAccountId: newOrExistingConnectedAccountId,
handle,
visibility:
calendarVisibility || CalendarChannelVisibility.SHARE_EVERYTHING,
},
{},
manager,
);
} else {
await connectedAccountRepository.update(
{
id: newOrExistingConnectedAccountId,
},
{
accessToken: input.accessToken,
refreshToken: input.refreshToken,
scopes,
},
manager,
);
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);
const workspaceMember = await workspaceMemberRepository.findOneOrFail({
where: { id: workspaceMemberId },
});
const userId = workspaceMember.userId;
await this.accountsToReconnectService.removeAccountToReconnect(
userId,
workspaceId,
newOrExistingConnectedAccountId,
);
// TODO: Modify this when the email sync is implemented
await messageChannelRepository.update(
{
connectedAccountId: newOrExistingConnectedAccountId,
},
{
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
syncCursor: '',
syncStageStartedAt: null,
},
manager,
);
}
});
// TODO: Uncomment this when the email sync is implemented
// const messageChannels = await messageChannelRepository.find({
// where: {
// connectedAccountId: newOrExistingConnectedAccountId,
// },
// });
// for (const messageChannel of messageChannels) {
// await this.messageQueueService.add<MessagingMessageListFetchJobData>(
// MessagingMessageListFetchJob.name,
// {
// workspaceId,
// messageChannelId: messageChannel.id,
// },
// );
// }
const calendarChannels = await calendarChannelRepository.find({
where: {
connectedAccountId: newOrExistingConnectedAccountId,
},
});
for (const calendarChannel of calendarChannels) {
await this.calendarQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,
workspaceId,
},
);
}
}
}

31
packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft-apis-oauth-common.auth.strategy.ts Normal file
View File

@ -0,0 +1,31 @@
import { Injectable } from '@nestjs/common';
import { PassportStrategy } from '@nestjs/passport';
import { Strategy } from 'passport-microsoft';
import { getMicrosoftApisOauthScopes } from 'src/engine/core-modules/auth/utils/get-microsoft-apis-oauth-scopes';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
export type MicrosoftAPIScopeConfig = {
isCalendarEnabled?: boolean;
isMessagingAliasFetchingEnabled?: boolean;
};
@Injectable()
export class MicrosoftAPIsOauthCommonStrategy extends PassportStrategy(
Strategy,
'microsoft-apis',
) {
constructor(environmentService: EnvironmentService) {
const scopes = getMicrosoftApisOauthScopes();
super({
clientID: environmentService.get('AUTH_MICROSOFT_CLIENT_ID'),
clientSecret: environmentService.get('AUTH_MICROSOFT_CLIENT_SECRET'),
tenant: environmentService.get('AUTH_MICROSOFT_TENANT_ID'),
callbackURL: environmentService.get('AUTH_MICROSOFT_APIS_CALLBACK_URL'),
scope: scopes,
passReqToCallback: true,
});
}
}

48
packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft-apis-oauth-exchange-code-for-token.auth.strategy.ts Normal file
View File

@ -0,0 +1,48 @@
import { Injectable } from '@nestjs/common';
import { VerifyCallback } from 'passport-google-oauth20';
import { MicrosoftAPIsOauthCommonStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-common.auth.strategy';
import { MicrosoftAPIsRequest } from 'src/engine/core-modules/auth/types/microsoft-api-request.type';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
export type MicrosoftAPIScopeConfig = {
isCalendarEnabled?: boolean;
};
@Injectable()
export class MicrosoftAPIsOauthExchangeCodeForTokenStrategy extends MicrosoftAPIsOauthCommonStrategy {
constructor(environmentService: EnvironmentService) {
super(environmentService);
}
async validate(
request: MicrosoftAPIsRequest,
accessToken: string,
refreshToken: string,
profile: any,
done: VerifyCallback,
): Promise<void> {
const { name, emails, photos } = profile;
const state =
typeof request.query.state === 'string'
? JSON.parse(request.query.state)
: undefined;
const user: MicrosoftAPIsRequest['user'] = {
emails,
firstName: name.givenName,
lastName: name.familyName,
picture: photos?.[0]?.value,
accessToken,
refreshToken,
transientToken: state.transientToken,
redirectLocation: state.redirectLocation,
calendarVisibility: state.calendarVisibility,
messageVisibility: state.messageVisibility,
};
done(null, user);
}
}

28
packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft-apis-oauth-request-code.auth.strategy.ts Normal file
View File

@ -0,0 +1,28 @@
import { Injectable } from '@nestjs/common';
import { MicrosoftAPIsOauthCommonStrategy } from 'src/engine/core-modules/auth/strategies/microsoft-apis-oauth-common.auth.strategy';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
@Injectable()
export class MicrosoftAPIsOauthRequestCodeStrategy extends MicrosoftAPIsOauthCommonStrategy {
constructor(environmentService: EnvironmentService) {
super(environmentService);
}
authenticate(req: any, options: any) {
options = {
...options,
accessType: 'offline',
prompt: 'consent',
loginHint: req.params.loginHint,
state: JSON.stringify({
transientToken: req.params.transientToken,
redirectLocation: req.params.redirectLocation,
calendarVisibility: req.params.calendarVisibility,
messageVisibility: req.params.messageVisibility,
}),
};
return super.authenticate(req, options);
}
}

23
packages/twenty-server/src/engine/core-modules/auth/types/microsoft-api-request.type.ts Normal file
View File

@ -0,0 +1,23 @@
import { Request } from 'express';
import { CalendarChannelVisibility } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { MessageChannelVisibility } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
export type MicrosoftAPIsRequest = Omit<
Request,
'user' | 'workspace' | 'workspaceMetadataVersion'
> & {
user: {
firstName?: string | null;
lastName?: string | null;
emails: { value: string }[];
picture: string | null;
workspaceInviteHash?: string;
accessToken: string;
refreshToken: string;
transientToken: string;
redirectLocation?: string;
calendarVisibility?: CalendarChannelVisibility;
messageVisibility?: MessageChannelVisibility;
};
};

12
packages/twenty-server/src/engine/core-modules/auth/utils/get-microsoft-apis-oauth-scopes.ts Normal file
View File

@ -0,0 +1,12 @@
export const getMicrosoftApisOauthScopes = () => {
const scopes = [
'openid',
'email',
'profile',
'offline_access',
'Mail.Read',
'Calendars.Read',
];
return scopes;
};

4
packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts
View File

@ -201,6 +201,10 @@ export class EnvironmentVariables {
@ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED)
AUTH_MICROSOFT_CALLBACK_URL: string;
@IsUrl({ require_tld: false })
@ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED)
AUTH_MICROSOFT_APIS_CALLBACK_URL: string;
@CastToBoolean()
@IsOptional()
@IsBoolean()

1
packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts
View File

@ -14,5 +14,6 @@ export enum FeatureFlagKey {
IsGmailSendEmailScopeEnabled = 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED',
IsAnalyticsV2Enabled = 'IS_ANALYTICS_V2_ENABLED',
IsUniqueIndexesEnabled = 'IS_UNIQUE_INDEXES_ENABLED',
IsMicrosoftSyncEnabled = 'IS_MICROSOFT_SYNC_ENABLED',
IsAdvancedFiltersEnabled = 'IS_ADVANCED_FILTERS_ENABLED',
}

1
packages/twenty-server/src/engine/core-modules/user/user.resolver.ts
View File

@ -35,7 +35,6 @@ import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorat
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { isDefined } from 'src/utils/is-defined';
const getHMACKey = (email?: string, key?: string | null) => {
if (!email || !key) return null;

12
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts
View File

@ -10,14 +10,19 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module';
import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command';
import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command';
import { CalendarOngoingStaleCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-ongoing-stale.cron.command';
import { CalendarEventListFetchCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job';
import { CalendarEventsImportCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job';
import { CalendarOngoingStaleCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-ongoing-stale.cron.job';
import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module';
import { MicrosoftCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/microsoft-calendar-driver.module';
import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarEventsImportJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job';
import { CalendarOngoingStaleJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job';
import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
@ -39,6 +44,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
WorkspaceDataSourceModule,
CalendarEventCleanerModule,
GoogleCalendarDriverModule,
MicrosoftCalendarDriverModule,
BillingModule,
RefreshAccessTokenManagerModule,
CalendarEventParticipantManagerModule,
@ -48,16 +54,20 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
providers: [
CalendarChannelSyncStatusService,
CalendarEventsImportService,
CalendarFetchEventsService,
CalendarEventImportErrorHandlerService,
CalendarGetCalendarEventsService,
CalendarSaveEventsService,
CalendarEventListFetchCronJob,
CalendarEventListFetchCronCommand,
CalendarEventListFetchJob,
CalendarEventsImportCronJob,
CalendarEventsImportCronCommand,
CalendarEventsImportJob,
CalendarOngoingStaleCronJob,
CalendarOngoingStaleCronCommand,
CalendarOngoingStaleJob,
],
exports: [CalendarEventsImportService],
exports: [CalendarEventsImportService, CalendarFetchEventsService],
})
export class CalendarEventImportManagerModule {}

1
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-event-import-batch-size.ts Normal file
View File

@ -0,0 +1 @@
export const CALENDAR_EVENT_IMPORT_BATCH_SIZE = 100;

6
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command.ts
View File

@ -3,10 +3,8 @@ import { Command, CommandRunner } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import {
CALENDAR_EVENTS_IMPORT_CRON_PATTERN,
CalendarEventListFetchCronJob,
} from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job';
import { CalendarEventListFetchCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job';
import { CALENDAR_EVENTS_IMPORT_CRON_PATTERN } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job';
@Command({
name: 'cron:calendar:calendar-event-list-fetch',

32
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command.ts Normal file
View File

@ -0,0 +1,32 @@
import { Command, CommandRunner } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import {
CALENDAR_EVENTS_IMPORT_CRON_PATTERN,
CalendarEventsImportCronJob,
} from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job';
@Command({
name: 'cron:calendar:calendar-events-import',
description: 'Starts a cron job to import the calendar events',
})
export class CalendarEventsImportCronCommand extends CommandRunner {
constructor(
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();
}
async run(): Promise<void> {
await this.messageQueueService.addCron<undefined>(
CalendarEventsImportCronJob.name,
undefined,
{
repeat: { pattern: CALENDAR_EVENTS_IMPORT_CRON_PATTERN },
},
);
}
}

8
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts
View File

@ -16,11 +16,11 @@ import {
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import {
CalendarEventListFetchJob,
CalendarEventsImportJobData,
CalendarEventListFetchJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarChannelSyncStage } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export const CALENDAR_EVENTS_IMPORT_CRON_PATTERN = '*/5 * * * *';
export const CALENDAR_EVENT_LIST_FETCH_CRON_PATTERN = '*/5 * * * *';
@Processor({
queueName: MessageQueue.cronQueue,
@ -38,7 +38,7 @@ export class CalendarEventListFetchCronJob {
@Process(CalendarEventListFetchCronJob.name)
@SentryCronMonitor(
CalendarEventListFetchCronJob.name,
CALENDAR_EVENTS_IMPORT_CRON_PATTERN,
CALENDAR_EVENT_LIST_FETCH_CRON_PATTERN,
)
async handle(): Promise<void> {
console.time('CalendarEventListFetchCronJob time');
@ -68,7 +68,7 @@ export class CalendarEventListFetchCronJob {
});
for (const calendarChannel of calendarChannels) {
await this.messageQueueService.add<CalendarEventsImportJobData>(
await this.messageQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,

87
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job.ts Normal file
View File

@ -0,0 +1,87 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Equal, Repository } from 'typeorm';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import {
Workspace,
WorkspaceActivationStatus,
} from 'src/engine/core-modules/workspace/workspace.entity';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { CalendarEventListFetchJobData } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarEventsImportJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job';
import { CalendarChannelSyncStage } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export const CALENDAR_EVENTS_IMPORT_CRON_PATTERN = '*/1 * * * *';
@Processor({
queueName: MessageQueue.cronQueue,
})
export class CalendarEventsImportCronJob {
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
@InjectMessageQueue(MessageQueue.calendarQueue)
private readonly messageQueueService: MessageQueueService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@Process(CalendarEventsImportCronJob.name)
@SentryCronMonitor(
CalendarEventsImportCronJob.name,
CALENDAR_EVENTS_IMPORT_CRON_PATTERN,
)
async handle(): Promise<void> {
console.time('CalendarEventsImportCronJob time');
const activeWorkspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
},
});
for (const activeWorkspace of activeWorkspaces) {
try {
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
activeWorkspace.id,
'calendarChannel',
);
const calendarChannels = await calendarChannelRepository.find({
where: {
isSyncEnabled: true,
syncStage: Equal(
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
),
},
});
for (const calendarChannel of calendarChannels) {
await this.messageQueueService.add<CalendarEventListFetchJobData>(
CalendarEventsImportJob.name,
{
calendarChannelId: calendarChannel.id,
workspaceId: activeWorkspace.id,
},
);
}
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {
user: {
workspaceId: activeWorkspace.id,
},
});
}
}
console.timeEnd('CalendarEventsImportCronJob time');
}
}

1
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts
View File

@ -72,6 +72,7 @@ export class GoogleCalendarGetEventsService {
}
return {
fullEvents: true,
calendarEvents: formatGoogleCalendarEvents(events),
nextSyncCursor: nextSyncToken || '',
};

20
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/microsoft-calendar-driver.module.ts Normal file
View File

@ -0,0 +1,20 @@
import { Module } from '@nestjs/common';
import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module';
import { MicrosoftCalendarGetEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service';
import { MicrosoftCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service';
import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
@Module({
imports: [EnvironmentModule],
providers: [
MicrosoftCalendarGetEventsService,
MicrosoftCalendarImportEventsService,
MicrosoftOAuth2ClientManagerService,
],
exports: [
MicrosoftCalendarGetEventsService,
MicrosoftCalendarImportEventsService,
],
})
export class MicrosoftCalendarDriverModule {}

62
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts Normal file
View File

@ -0,0 +1,62 @@
import { Injectable } from '@nestjs/common';
import {
PageCollection,
PageIterator,
PageIteratorCallback,
} from '@microsoft/microsoft-graph-client';
import { parseMicrosoftCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/parse-microsoft-calendar-error.util';
import { GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class MicrosoftCalendarGetEventsService {
constructor(
private readonly microsoftOAuth2ClientManagerService: MicrosoftOAuth2ClientManagerService,
) {}
public async getCalendarEvents(
connectedAccount: Pick<
ConnectedAccountWorkspaceEntity,
'provider' | 'refreshToken' | 'id'
>,
syncCursor?: string,
): Promise<GetCalendarEventsResponse> {
try {
const microsoftClient =
await this.microsoftOAuth2ClientManagerService.getOAuth2Client(
connectedAccount.refreshToken,
);
const eventIds: string[] = [];
const response: PageCollection = await microsoftClient
.api(syncCursor || '/me/calendar/events/delta')
.version('beta')
.get();
const callback: PageIteratorCallback = (data) => {
eventIds.push(data.id);
return true;
};
const pageIterator = new PageIterator(
microsoftClient,
response,
callback,
);
await pageIterator.iterate();
return {
fullEvents: false,
calendarEventIds: eventIds,
nextSyncCursor: pageIterator.getDeltaLink() || '',
};
} catch (error) {
throw parseMicrosoftCalendarError(error);
}
}
}

45
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts Normal file
View File

@ -0,0 +1,45 @@
import { Injectable } from '@nestjs/common';
import { Event } from '@microsoft/microsoft-graph-types';
import { formatMicrosoftCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util';
import { parseMicrosoftCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/parse-microsoft-calendar-error.util';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class MicrosoftCalendarImportEventsService {
constructor(
private readonly microsoftOAuth2ClientManagerService: MicrosoftOAuth2ClientManagerService,
) {}
public async getCalendarEvents(
connectedAccount: Pick<
ConnectedAccountWorkspaceEntity,
'provider' | 'refreshToken' | 'id'
>,
changedEventIds: string[],
): Promise<CalendarEventWithParticipants[]> {
try {
const microsoftClient =
await this.microsoftOAuth2ClientManagerService.getOAuth2Client(
connectedAccount.refreshToken,
);
const events: Event[] = [];
for (const changedEventId of changedEventIds) {
const event = await microsoftClient
.api(`/me/calendar/events/${changedEventId}`)
.get();
events.push(event);
}
return formatMicrosoftCalendarEvents(events);
} catch (error) {
throw parseMicrosoftCalendarError(error);
}
}
}

60
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util.ts Normal file
View File

@ -0,0 +1,60 @@
import {
Event,
NullableOption,
ResponseType,
} from '@microsoft/microsoft-graph-types';
import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
export const formatMicrosoftCalendarEvents = (
events: Event[],
): CalendarEventWithParticipants[] => {
return events.map(formatMicrosoftCalendarEvent);
};
const formatMicrosoftCalendarEvent = (
event: Event,
): CalendarEventWithParticipants => {
const formatResponseStatus = (
status: NullableOption<ResponseType> | undefined,
) => {
switch (status) {
case 'accepted':
case 'organizer':
return CalendarEventParticipantResponseStatus.ACCEPTED;
case 'declined':
return CalendarEventParticipantResponseStatus.DECLINED;
case 'tentativelyAccepted':
return CalendarEventParticipantResponseStatus.TENTATIVE;
default:
return CalendarEventParticipantResponseStatus.NEEDS_ACTION;
}
};
return {
title: event.subject ?? '',
isCanceled: !!event.isCancelled,
isFullDay: !!event.isAllDay,
startsAt: event.start?.dateTime ?? null,
endsAt: event.end?.dateTime ?? null,
externalId: event.id ?? '',
externalCreatedAt: event.createdDateTime ?? null,
externalUpdatedAt: event.lastModifiedDateTime ?? null,
description: event.body?.content ?? '',
location: event.location?.displayName ?? '',
iCalUID: event.iCalUId ?? '',
conferenceSolution: event.onlineMeetingProvider ?? '',
conferenceLinkLabel: event.onlineMeeting?.joinUrl ?? '',
conferenceLinkUrl: event.onlineMeeting?.joinUrl ?? '',
recurringEventExternalId: event.id ?? '',
participants:
event.attendees?.map((attendee) => ({
handle: attendee.emailAddress?.address ?? '',
displayName: attendee.emailAddress?.name ?? '',
isOrganizer: attendee.status?.response === 'organizer',
responseStatus: formatResponseStatus(attendee.status?.response),
})) ?? [],
status: '',
};
};

65
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/parse-microsoft-calendar-error.util.ts Normal file
View File

@ -0,0 +1,65 @@
import { GraphError } from '@microsoft/microsoft-graph-client';
import {
CalendarEventImportDriverException,
CalendarEventImportDriverExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception';
export const parseMicrosoftCalendarError = (
error: GraphError,
): CalendarEventImportDriverException => {
const { statusCode, message } = error;
switch (statusCode) {
case 400:
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
case 404:
if (
message ==
'The mailbox is either inactive, soft-deleted, or is hosted on-premise.'
) {
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS,
);
}
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.NOT_FOUND,
);
case 429:
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.TEMPORARY_ERROR,
);
case 403:
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS,
);
case 401:
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS,
);
case 500:
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
default:
return new CalendarEventImportDriverException(
message,
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
}
};

12
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts
View File

@ -4,14 +4,14 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
export type CalendarEventsImportJobData = {
export type CalendarEventListFetchJobData = {
calendarChannelId: string;
workspaceId: string;
};
@ -23,11 +23,11 @@ export type CalendarEventsImportJobData = {
export class CalendarEventListFetchJob {
constructor(
private readonly twentyORMManager: TwentyORMManager,
private readonly calendarEventsImportService: CalendarEventsImportService,
private readonly calendarFetchEventsService: CalendarFetchEventsService,
) {}
@Process(CalendarEventListFetchJob.name)
async handle(data: CalendarEventsImportJobData): Promise<void> {
async handle(data: CalendarEventListFetchJobData): Promise<void> {
console.time('CalendarEventListFetchJob time');
const { workspaceId, calendarChannelId } = data;
@ -65,7 +65,7 @@ export class CalendarEventListFetchJob {
syncStageStartedAt: null,
});
await this.calendarEventsImportService.processCalendarEventsImport(
await this.calendarFetchEventsService.fetchCalendarEvents(
calendarChannel,
calendarChannel.connectedAccount,
workspaceId,
@ -73,7 +73,7 @@ export class CalendarEventListFetchJob {
break;
case CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING:
await this.calendarEventsImportService.processCalendarEventsImport(
await this.calendarFetchEventsService.fetchCalendarEvents(
calendarChannel,
calendarChannel.connectedAccount,
workspaceId,

75
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts Normal file
View File

@ -0,0 +1,75 @@
import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
export type CalendarEventsImportJobData = {
calendarChannelId: string;
workspaceId: string;
};
@Processor({
queueName: MessageQueue.calendarQueue,
scope: Scope.REQUEST,
})
export class CalendarEventsImportJob {
constructor(
private readonly calendarEventsImportService: CalendarEventsImportService,
private readonly twentyORMManager: TwentyORMManager,
) {}
@Process(CalendarEventsImportJob.name)
async handle(data: CalendarEventsImportJobData): Promise<void> {
console.time('CalendarEventsImportJob time');
const { calendarChannelId, workspaceId } = data;
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
},
relations: ['connectedAccount'],
});
if (!calendarChannel?.isSyncEnabled) {
return;
}
if (
isThrottled(
calendarChannel.syncStageStartedAt,
calendarChannel.throttleFailureCount,
)
) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING
) {
return;
}
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel,
calendarChannel.connectedAccount,
workspaceId,
);
console.timeEnd('CalendarEventsImportJob time');
}
}

87
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts
View File

@ -2,84 +2,86 @@ import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
import { CALENDAR_EVENT_IMPORT_BATCH_SIZE } from 'src/modules/calendar/calendar-event-import-manager/constants/calendar-event-import-batch-size';
import { MicrosoftCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service';
import {
CalendarEventImportErrorHandlerService,
CalendarEventImportSyncStep,
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
import {
CalendarGetCalendarEventsService,
GetCalendarEventsResponse,
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class CalendarEventsImportService {
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly twentyORMManager: TwentyORMManager,
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly calendarEventCleanerService: CalendarEventCleanerService,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly getCalendarEventsService: CalendarGetCalendarEventsService,
private readonly calendarSaveEventsService: CalendarSaveEventsService,
private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService,
private readonly microsoftCalendarImportEventService: MicrosoftCalendarImportEventsService,
) {}
public async processCalendarEventsImport(
calendarChannel: CalendarChannelWorkspaceEntity,
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
fetchedCalendarEvents?: CalendarEventWithParticipants[],
): Promise<void> {
const syncStep =
calendarChannel.syncStage ===
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING
? CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH
: CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH;
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing(
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportOngoing(
[calendarChannel.id],
);
let calendarEvents: GetCalendarEventsResponse['calendarEvents'] = [];
let nextSyncCursor: GetCalendarEventsResponse['nextSyncCursor'] = '';
let calendarEvents: CalendarEventWithParticipants[] = [];
try {
const getCalendarEventsResponse =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccount,
calendarChannel.syncCursor,
if (fetchedCalendarEvents) {
calendarEvents = fetchedCalendarEvents;
} else {
const eventIdsToFetch: string[] = await this.cacheStorage.setPop(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
CALENDAR_EVENT_IMPORT_BATCH_SIZE,
);
calendarEvents = getCalendarEventsResponse.calendarEvents;
nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
if (!eventIdsToFetch || eventIdsToFetch.length === 0) {
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialCalendarEventListFetch(
[calendarChannel.id],
);
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
return;
}
switch (connectedAccount.provider) {
case 'microsoft':
calendarEvents =
await this.microsoftCalendarImportEventService.getCalendarEvents(
connectedAccount,
eventIdsToFetch,
);
break;
default:
break;
}
}
if (!calendarEvents || calendarEvents?.length === 0) {
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
[calendarChannel.id],
);
@ -127,22 +129,13 @@ export class CalendarEventsImportService {
workspaceId,
);
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialCalendarEventListFetch(
[calendarChannel.id],
);
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
syncStep,
CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT,
calendarChannel,
workspaceId,
);

129
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts Normal file
View File

@ -0,0 +1,129 @@
import { Injectable } from '@nestjs/common';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {
CalendarEventImportDriverException,
CalendarEventImportDriverExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception';
import {
CalendarEventImportErrorHandlerService,
CalendarEventImportSyncStep,
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class CalendarFetchEventsService {
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly twentyORMManager: TwentyORMManager,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly getCalendarEventsService: CalendarGetCalendarEventsService,
private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService,
private readonly calendarEventsImportService: CalendarEventsImportService,
) {}
public async fetchCalendarEvents(
calendarChannel: CalendarChannelWorkspaceEntity,
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
): Promise<void> {
const syncStep =
calendarChannel.syncStage ===
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING
? CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH
: CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH;
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing(
[calendarChannel.id],
);
try {
const getCalendarEventsResponse =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccount,
calendarChannel.syncCursor,
);
const hasFullEvents = getCalendarEventsResponse.fullEvents;
const calendarEvents = hasFullEvents
? getCalendarEventsResponse.calendarEvents
: null;
const calendarEventIds = getCalendarEventsResponse.calendarEventIds;
const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
if (!calendarEvents || calendarEvents?.length === 0) {
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
[calendarChannel.id],
);
}
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
if (hasFullEvents && calendarEvents) {
// Event Import already done
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel,
connectedAccount,
workspaceId,
calendarEvents,
);
} else if (!hasFullEvents && calendarEventIds) {
// Event Import still needed
await this.cacheStorage.setAdd(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
calendarEventIds,
);
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
[calendarChannel.id],
);
} else {
throw new CalendarEventImportDriverException(
"Expected 'calendarEvents' or 'calendarEventIds' to be present",
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
}
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
syncStep,
calendarChannel,
workspaceId,
);
}
}
}

17
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service.ts
View File

@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { GoogleCalendarGetEventsService as GoogleCalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service';
import { GoogleCalendarGetEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service';
import { MicrosoftCalendarGetEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service';
import {
CalendarEventImportException,
CalendarEventImportExceptionCode,
@ -9,14 +10,17 @@ import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
export type GetCalendarEventsResponse = {
calendarEvents: CalendarEventWithParticipants[];
fullEvents: boolean;
calendarEvents?: CalendarEventWithParticipants[];
calendarEventIds?: string[];
nextSyncCursor: string;
};
@Injectable()
export class CalendarGetCalendarEventsService {
constructor(
private readonly googleCalendarGetCalendarEventsService: GoogleCalendarGetCalendarEventsService,
private readonly googleCalendarGetEventsService: GoogleCalendarGetEventsService,
private readonly microsoftCalendarGetEventsService: MicrosoftCalendarGetEventsService,
) {}
public async getCalendarEvents(
@ -28,7 +32,12 @@ export class CalendarGetCalendarEventsService {
): Promise<GetCalendarEventsResponse> {
switch (connectedAccount.provider) {
case 'google':
return this.googleCalendarGetCalendarEventsService.getCalendarEvents(
return this.googleCalendarGetEventsService.getCalendarEvents(
connectedAccount,
syncCursor,
);
case 'microsoft':
return this.microsoftCalendarGetEventsService.getCalendarEvents(
connectedAccount,
syncCursor,
);

2
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/utils/filter-events.util.ts
View File

@ -23,7 +23,7 @@ export const filterEventsAndReturnCancelledEvents = (
},
event,
) => {
if (event.status === 'cancelled') {
if (event.isCanceled) {
acc.cancelledEvents.push(event);
} else {
acc.filteredEvents.push(event);

62
packages/twenty-server/src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service.ts Normal file
View File

@ -0,0 +1,62 @@
import { Injectable } from '@nestjs/common';
import {
AuthProvider,
AuthProviderCallback,
Client,
} from '@microsoft/microsoft-graph-client';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
@Injectable()
export class MicrosoftOAuth2ClientManagerService {
constructor(private readonly environmentService: EnvironmentService) {}
public async getOAuth2Client(refreshToken: string): Promise<Client> {
const authProvider: AuthProvider = async (
callback: AuthProviderCallback,
) => {
try {
const tenantId = this.environmentService.get(
'AUTH_MICROSOFT_TENANT_ID',
);
const urlData = new URLSearchParams();
urlData.append(
'client_id',
this.environmentService.get('AUTH_MICROSOFT_CLIENT_ID'),
);
urlData.append('scope', 'https://graph.microsoft.com/.default');
urlData.append('refresh_token', refreshToken);
urlData.append(
'client_secret',
this.environmentService.get('AUTH_MICROSOFT_CLIENT_SECRET'),
);
urlData.append('grant_type', 'refresh_token');
const res = await fetch(
`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`,
{
method: 'POST',
body: urlData,
},
);
const data = await res.json();
callback(null, data.access_token);
} catch (error) {
callback(error, null);
}
};
const client = Client.init({
defaultVersion: 'v1.0',
debugLogging: false,
authProvider: authProvider,
});
return client;
}
}

9
packages/twenty-server/src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module.ts
View File

@ -1,11 +1,16 @@
import { Module } from '@nestjs/common';
import { GoogleOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/google/google-oauth2-client-manager.service';
import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
@Module({
imports: [],
providers: [OAuth2ClientManagerService, GoogleOAuth2ClientManagerService],
exports: [OAuth2ClientManagerService],
providers: [
OAuth2ClientManagerService,
GoogleOAuth2ClientManagerService,
MicrosoftOAuth2ClientManagerService,
],
exports: [OAuth2ClientManagerService, MicrosoftOAuth2ClientManagerService],
})
export class OAuth2ClientManagerModule {}

1
packages/twenty-server/src/modules/connected-account/standard-objects/connected-account.workspace-entity.ts
View File

@ -22,6 +22,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
export enum ConnectedAccountProvider {
GOOGLE = 'google',
MICROSOFT = 'microsoft',
}
@WorkspaceEntity({

12
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts
View File

@ -35,6 +35,12 @@ export class MessagingGetMessageListService {
return this.gmailGetMessageListService.getFullMessageList(
connectedAccount,
);
case 'microsoft':
// TODO: Placeholder
return {
messageExternalIds: [],
nextSyncCursor: '',
};
default:
throw new MessageImportException(
`Provider ${connectedAccount.provider} is not supported`,
@ -56,6 +62,12 @@ export class MessagingGetMessageListService {
connectedAccount,
syncCursor,
);
case 'microsoft':
return {
messageExternalIds: [],
messageExternalIdsToDelete: [],
nextSyncCursor: '',
};
default:
throw new MessageImportException(
`Provider ${connectedAccount.provider} is not supported`,