4285 timebox create google calendar full sync (#4442)

* calendar module

* wip

* creating a folder for common files between calendar and messages

* wip

* wip

* wip

* wip

* update calendar search filter

* wip

* working on full sync service

* reorganizing folders

* adding repositories

* fix typo

* working on full-sync service

* Add calendarQueue to MessageQueue enum and update dependencies

* start transaction

* wip

* add save and update functions for event

* wip

* save events

* improving step by step

* add calendar scope

* fix nest modules imports

* renaming

* create calendar channel

* create job for google calendar full-sync

* call GoogleCalendarFullSyncJob after connected account creation

* ask for scope conditionnally

* fixes

* create channels conditionnally

* fix

* fixes

* fix FK bug

* filter out canceled events

* create save and update functions for calendarEventAttendee repository

* saving messageParticipants is working

* save calendarEventAttendees is working

* add calendarEvent cleaner

* calendar event cleaner is working

* working on updating attendees

* wip

* reintroducing google-gmail endpoint to ensure smooth deploy

* modify callbackURL

* modify front url

* changes to be able to merge

* put back feature flag

* fixes after PR comments

* add feature flag check

* remove unused modules

* separate delete connected account associated job data in two jobs

* fix error

* rename calendar_v3 as calendarV3

* Update packages/twenty-server/src/workspace/calendar-and-messaging/utils/valueStringForBatchRawQuery.util.ts

Co-authored-by: Jérémy M <jeremy.magrin@gmail.com>

* improve readability

* renaming to remove plural

* renaming to remove plural

* don't throw if no connected account is found

* use calendar queue

* modify usage of HttpService in fetch-by-batch

* modify valuesStringForBatchRawQuery to improve api and return flattened values

* fix auth module feature flag import

* fix getFlattenedValuesAndValuesStringForBatchRawQuery

---------

Co-authored-by: Jérémy M <jeremy.magrin@gmail.com>
This commit is contained in:
bosiraphael
2024-03-14 11:23:31 +01:00
committed by GitHub
parent e0dac82e07
commit 3caf860848
76 changed files with 1856 additions and 280 deletions

View File

@ -0,0 +1,21 @@
import { Module } from '@nestjs/common';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service';
import { CreateCompanyModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.module';
import { CreateContactModule } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.module';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
WorkspaceDataSourceModule,
CreateContactModule,
CreateCompanyModule,
WorkspaceMemberModule,
PersonModule,
],
providers: [CreateCompanyAndContactService],
exports: [CreateCompanyAndContactService],
})
export class CreateCompaniesAndContactsModule {}

View File

@ -5,16 +5,16 @@ import compact from 'lodash/compact';
import { Participant } from 'src/workspace/messaging/types/gmail-message';
import { getDomainNameFromHandle } from 'src/workspace/messaging/utils/get-domain-name-from-handle.util';
import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service';
import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service';
import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service';
import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service';
import { getUniqueParticipantsAndHandles } from 'src/workspace/messaging/utils/get-unique-participants-and-handles.util';
import { filterOutParticipantsFromCompanyOrWorkspace } from 'src/workspace/messaging/utils/filter-out-participants-from-company-or-workspace.util';
import { isWorkEmail } from 'src/utils/is-work-email';
@Injectable()
export class CreateCompaniesAndContactsService {
export class CreateCompanyAndContactService {
constructor(
private readonly personService: PersonService,
private readonly createContactService: CreateContactService,

View File

@ -1,7 +1,7 @@
import { Module } from '@nestjs/common';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service';
import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service';
import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module';
@Module({

View File

@ -1,8 +1,8 @@
import { Module } from '@nestjs/common';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
@Module({
imports: [WorkspaceDataSourceModule, PersonModule],

View File

@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
import { getFirstNameAndLastNameFromHandleAndDisplayName } from 'src/workspace/messaging/utils/get-first-name-and-last-name-from-handle-and-display-name.util';
type ContactToCreate = {

View File

@ -1,6 +1,6 @@
import { Module } from '@nestjs/common';
import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({

View File

@ -1,6 +1,6 @@
import { Module } from '@nestjs/common';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({

View File

@ -3,10 +3,10 @@ import { Injectable } from '@nestjs/common';
import axios from 'axios';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
@Injectable()
export class GmailRefreshAccessTokenService {
export class GoogleAPIsRefreshAccessTokenService {
constructor(
private readonly environmentService: EnvironmentService,
private readonly connectedAccountService: ConnectedAccountService,

View File

@ -0,0 +1,5 @@
type Query = {
uri: string;
};
export type BatchQueries = Query[];

View File

@ -0,0 +1,56 @@
export const valuesStringForBatchRawQuery = (
values: {
[key: string]: any;
}[],
typesArray: string[] = [],
) => {
const castedValues = values.reduce((acc, _, rowIndex) => {
const numberOfColumns = typesArray.length;
const rowValues = Array.from(
{ length: numberOfColumns },
(_, columnIndex) => {
const placeholder = `$${rowIndex * numberOfColumns + columnIndex + 1}`;
const typeCast = typesArray[columnIndex]
? `::${typesArray[columnIndex]}`
: '';
return `${placeholder}${typeCast}`;
},
).join(', ');
acc.push(`(${rowValues})`);
return acc;
}, [] as string[]);
return castedValues.join(', ');
};
export const getFlattenedValuesAndValuesStringForBatchRawQuery = (
values: {
[key: string]: any;
}[],
keyTypeMap: {
[key: string]: string;
},
): {
flattenedValues: any[];
valuesString: string;
} => {
const keysToInsert = Object.keys(keyTypeMap);
const flattenedValues = values.flatMap((value) =>
keysToInsert.map((key) => value[key]),
);
const valuesString = valuesStringForBatchRawQuery(
values,
Object.values(keyTypeMap),
);
return {
flattenedValues,
valuesString,
};
};

View File

@ -0,0 +1,41 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { EnvironmentModule } from 'src/integrations/environment/environment.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
import { BlocklistModule } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { CalendarChannelEventAssociationModule } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module';
import { CalendarChannelModule } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.module';
import { CalendarEventAttendeeModule } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module';
import { CalendarEventModule } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.module';
import { CalendarEventCleanerModule } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
import { GoogleCalendarFullSyncService } from 'src/workspace/calendar/services/google-calendar-full-sync.service';
import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider';
import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
EnvironmentModule,
WorkspaceDataSourceModule,
ConnectedAccountModule,
CalendarChannelModule,
CalendarChannelEventAssociationModule,
CalendarEventModule,
CalendarEventAttendeeModule,
CreateCompaniesAndContactsModule,
WorkspaceMemberModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
CompanyModule,
PersonModule,
BlocklistModule,
CalendarEventCleanerModule,
],
providers: [GoogleCalendarFullSyncService, GoogleCalendarClientProvider],
exports: [GoogleCalendarFullSyncService],
})
export class CalendarModule {}

View File

@ -0,0 +1,66 @@
import { Inject } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import {
GoogleCalendarFullSyncJobData,
GoogleCalendarFullSyncJob,
} from 'src/workspace/calendar/jobs/google-calendar-full-sync.job';
interface GoogleCalendarFullSyncOptions {
workspaceId: string;
}
@Command({
name: 'workspace:google-calendar-full-sync',
description:
'Start google calendar full-sync for all workspaceMembers in a workspace.',
})
export class GoogleCalendarFullSyncCommand extends CommandRunner {
constructor(
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
private readonly connectedAccountService: ConnectedAccountService,
) {
super();
}
async run(
_passedParam: string[],
options: GoogleCalendarFullSyncOptions,
): Promise<void> {
await this.fetchWorkspaceCalendars(options.workspaceId);
return;
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
private async fetchWorkspaceCalendars(workspaceId: string): Promise<void> {
const connectedAccounts =
await this.connectedAccountService.getAll(workspaceId);
for (const connectedAccount of connectedAccounts) {
await this.messageQueueService.add<GoogleCalendarFullSyncJobData>(
GoogleCalendarFullSyncJob.name,
{
workspaceId,
connectedAccountId: connectedAccount.id,
},
{
retryLimit: 2,
},
);
}
}
}

View File

@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { GoogleCalendarFullSyncCommand } from 'src/workspace/calendar/commands/google-calendar-full-sync.command';
@Module({
imports: [
DataSourceModule,
TypeORMModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
ConnectedAccountModule,
],
providers: [GoogleCalendarFullSyncCommand],
})
export class WorkspaceCalendarSyncCommandsModule {}

View File

@ -0,0 +1,53 @@
import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { GoogleCalendarFullSyncService } from 'src/workspace/calendar/services/google-calendar-full-sync.service';
export type GoogleCalendarFullSyncJobData = {
workspaceId: string;
connectedAccountId: string;
nextPageToken?: string;
};
@Injectable()
export class GoogleCalendarFullSyncJob
implements MessageQueueJob<GoogleCalendarFullSyncJobData>
{
private readonly logger = new Logger(GoogleCalendarFullSyncJob.name);
constructor(
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService,
private readonly googleCalendarFullSyncService: GoogleCalendarFullSyncService,
) {}
async handle(data: GoogleCalendarFullSyncJobData): Promise<void> {
this.logger.log(
`google calendar full-sync for workspace ${
data.workspaceId
} and account ${data.connectedAccountId} ${
data.nextPageToken ? `and ${data.nextPageToken} pageToken` : ''
}`,
);
try {
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
data.workspaceId,
data.connectedAccountId,
);
} catch (e) {
this.logger.error(
`Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
e,
);
return;
}
await this.googleCalendarFullSyncService.startGoogleCalendarFullSync(
data.workspaceId,
data.connectedAccountId,
data.nextPageToken,
);
}
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarChannelEventAssociationService } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarChannelEventAssociationService],
exports: [CalendarChannelEventAssociationService],
})
export class CalendarChannelEventAssociationModule {}

View File

@ -0,0 +1,172 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
import { CalendarChannelEventAssociationObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel-event-association.object-metadata';
import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util';
@Injectable()
export class CalendarChannelEventAssociationService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByEventExternalIdsAndCalendarChannelId(
eventExternalIds: string[],
calendarChannelId: string,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelEventAssociationObjectMetadata>[]> {
if (eventExternalIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation"
WHERE "eventExternalId" = ANY($1) AND "calendarChannelId" = $2`,
[eventExternalIds, calendarChannelId],
workspaceId,
transactionManager,
);
}
public async deleteByEventExternalIdsAndCalendarChannelId(
eventExternalIds: string[],
calendarChannelId: string,
workspaceId: string,
transactionManager?: EntityManager,
) {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "eventExternalId" = ANY($1) AND "calendarChannelId" = $2`,
[eventExternalIds, calendarChannelId],
workspaceId,
transactionManager,
);
}
public async getByCalendarChannelIds(
calendarChannelIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelEventAssociationObjectMetadata>[]> {
if (calendarChannelIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation"
WHERE "calendarChannelId" = ANY($1)`,
[calendarChannelIds],
workspaceId,
transactionManager,
);
}
public async deleteByCalendarChannelIds(
calendarChannelIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
) {
if (calendarChannelIds.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "calendarChannelId" = ANY($1)`,
[calendarChannelIds],
workspaceId,
transactionManager,
);
}
public async deleteByIds(
ids: string[],
workspaceId: string,
transactionManager?: EntityManager,
) {
if (ids.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "id" = ANY($1)`,
[ids],
workspaceId,
transactionManager,
);
}
public async getByCalendarEventIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelEventAssociationObjectMetadata>[]> {
if (calendarEventIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation"
WHERE "calendarEventId" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async saveCalendarChannelEventAssociations(
calendarChannelEventAssociations: Omit<
ObjectRecord<CalendarChannelEventAssociationObjectMetadata>,
'id' | 'createdAt' | 'updatedAt' | 'calendarChannel' | 'calendarEvent'
>[],
workspaceId: string,
transactionManager?: EntityManager,
) {
if (calendarChannelEventAssociations.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const {
flattenedValues: calendarChannelEventAssociationValues,
valuesString,
} = getFlattenedValuesAndValuesStringForBatchRawQuery(
calendarChannelEventAssociations,
{
calendarChannelId: 'uuid',
calendarEventId: 'uuid',
eventExternalId: 'text',
},
);
await this.workspaceDataSourceService.executeRawQuery(
`INSERT INTO ${dataSourceSchema}."calendarChannelEventAssociation" ("calendarChannelId", "calendarEventId", "eventExternalId")
VALUES ${valuesString}`,
calendarChannelEventAssociationValues,
workspaceId,
transactionManager,
);
}
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarChannelService } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarChannelService],
exports: [CalendarChannelService],
})
export class CalendarChannelModule {}

View File

@ -0,0 +1,76 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { CalendarChannelObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel.object-metadata';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
@Injectable()
export class CalendarChannelService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByConnectedAccountId(
connectedAccountId: string,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelObjectMetadata>[]> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannel" WHERE "connectedAccountId" = $1 LIMIT 1`,
[connectedAccountId],
workspaceId,
transactionManager,
);
}
public async getFirstByConnectedAccountIdOrFail(
connectedAccountId: string,
workspaceId: string,
): Promise<ObjectRecord<CalendarChannelObjectMetadata>> {
const calendarChannels = await this.getByConnectedAccountId(
connectedAccountId,
workspaceId,
);
if (!calendarChannels || calendarChannels.length === 0) {
throw new Error(
`No calendar channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
);
}
return calendarChannels[0];
}
public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail(
connectedAccountId: string,
workspaceId: string,
): Promise<boolean> {
const calendarChannel = await this.getFirstByConnectedAccountIdOrFail(
connectedAccountId,
workspaceId,
);
return calendarChannel.isContactAutoCreationEnabled;
}
public async getByIds(
ids: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelObjectMetadata>[]> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannel" WHERE "id" = ANY($1)`,
[ids],
workspaceId,
transactionManager,
);
}
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarEventAttendeeService } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarEventAttendeeService],
exports: [CalendarEventAttendeeService],
})
export class CalendarEventAttendeeModule {}

View File

@ -0,0 +1,151 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util';
import { CalendarEventAttendee } from 'src/workspace/calendar/types/calendar-event';
@Injectable()
export class CalendarEventAttendeeService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByIds(
calendarEventAttendeeIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventAttendeeObjectMetadata>[]> {
if (calendarEventAttendeeIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
[calendarEventAttendeeIds],
workspaceId,
transactionManager,
);
}
public async getByCalendarEventIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventAttendeeObjectMetadata>[]> {
if (calendarEventIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "calendarEventId" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async deleteByIds(
calendarEventAttendeeIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventAttendeeIds.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
[calendarEventAttendeeIds],
workspaceId,
transactionManager,
);
}
public async saveCalendarEventAttendees(
calendarEventAttendees: CalendarEventAttendee[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventAttendees.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(
calendarEventAttendees,
{
calendarEventId: 'uuid',
handle: 'text',
displayName: 'text',
isOrganizer: 'boolean',
responseStatus: `${dataSourceSchema}."calendarEventAttendee_responsestatus_enum"`,
},
);
await this.workspaceDataSourceService.executeRawQuery(
`INSERT INTO ${dataSourceSchema}."calendarEventAttendee" ("calendarEventId", "handle", "displayName", "isOrganizer", "responseStatus") VALUES ${valuesString}`,
flattenedValues,
workspaceId,
transactionManager,
);
}
public async updateCalendarEventAttendees(
calendarEventAttendees: CalendarEventAttendee[],
iCalUIDCalendarEventIdMap: Map<string, string>,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventAttendees.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const values = calendarEventAttendees.map((calendarEventAttendee) => ({
...calendarEventAttendee,
calendarEventId: iCalUIDCalendarEventIdMap.get(
calendarEventAttendee.iCalUID,
),
}));
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(values, {
calendarEventId: 'uuid',
handle: 'text',
displayName: 'text',
isOrganizer: 'boolean',
responseStatus: `${dataSourceSchema}."calendarEventAttendee_responsestatus_enum"`,
});
await this.workspaceDataSourceService.executeRawQuery(
`UPDATE ${dataSourceSchema}."calendarEventAttendee" AS "calendarEventAttendee"
SET "displayName" = "newValues"."displayName",
"isOrganizer" = "newValues"."isOrganizer",
"responseStatus" = "newValues"."responseStatus"
FROM (VALUES ${valuesString}) AS "newValues"("calendarEventId", "handle", "displayName", "isOrganizer", "responseStatus")
WHERE "calendarEventAttendee"."handle" = "newValues"."handle"
AND "calendarEventAttendee"."calendarEventId" = "newValues"."calendarEventId"`,
flattenedValues,
workspaceId,
transactionManager,
);
}
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarEventService],
exports: [CalendarEventService],
})
export class CalendarEventModule {}

View File

@ -0,0 +1,202 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
import { CalendarEventObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata';
import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util';
import { CalendarEvent } from 'src/workspace/calendar/types/calendar-event';
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
@Injectable()
export class CalendarEventService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventObjectMetadata>[]> {
if (calendarEventIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarEvent" WHERE "id" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async deleteByIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventIds.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarEvent" WHERE "id" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async getNonAssociatedCalendarEventIdsPaginated(
limit: number,
offset: number,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventAttendeeObjectMetadata>[]> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const nonAssociatedCalendarEvents =
await this.workspaceDataSourceService.executeRawQuery(
`SELECT m.id FROM ${dataSourceSchema}."calendarEvent" m
LEFT JOIN ${dataSourceSchema}."calendarChannelEventAssociation" ccea
ON m.id = ccea."calendarEventId"
WHERE ccea.id IS NULL
LIMIT $1 OFFSET $2`,
[limit, offset],
workspaceId,
transactionManager,
);
return nonAssociatedCalendarEvents.map(({ id }) => id);
}
public async getICalUIDCalendarEventIdMap(
iCalUIDs: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<Map<string, string>> {
if (iCalUIDs.length === 0) {
return new Map();
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const calendarEvents: {
id: string;
iCalUID: string;
}[] = await this.workspaceDataSourceService.executeRawQuery(
`SELECT id, "iCalUID" FROM ${dataSourceSchema}."calendarEvent" WHERE "iCalUID" = ANY($1)`,
[iCalUIDs],
workspaceId,
transactionManager,
);
const iCalUIDsCalendarEvnetIdsMap = new Map<string, string>();
calendarEvents.forEach((calendarEvent) => {
iCalUIDsCalendarEvnetIdsMap.set(calendarEvent.iCalUID, calendarEvent.id);
});
return iCalUIDsCalendarEvnetIdsMap;
}
public async saveCalendarEvents(
calendarEvents: CalendarEvent[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEvents.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(calendarEvents, {
id: 'uuid',
title: 'text',
isCanceled: 'boolean',
isFullDay: 'boolean',
startsAt: 'timestamptz',
endsAt: 'timestamptz',
externalCreatedAt: 'timestamptz',
externalUpdatedAt: 'timestamptz',
description: 'text',
location: 'text',
iCalUID: 'text',
conferenceSolution: 'text',
conferenceUri: 'text',
recurringEventExternalId: 'text',
});
await this.workspaceDataSourceService.executeRawQuery(
`INSERT INTO ${dataSourceSchema}."calendarEvent" ("id", "title", "isCanceled", "isFullDay", "startsAt", "endsAt", "externalCreatedAt", "externalUpdatedAt", "description", "location", "iCalUID", "conferenceSolution", "conferenceUri", "recurringEventExternalId") VALUES ${valuesString}`,
flattenedValues,
workspaceId,
transactionManager,
);
}
public async updateCalendarEvents(
calendarEvents: CalendarEvent[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEvents.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(calendarEvents, {
title: 'text',
isCanceled: 'boolean',
isFullDay: 'boolean',
startsAt: 'timestamptz',
endsAt: 'timestamptz',
externalCreatedAt: 'timestamptz',
externalUpdatedAt: 'timestamptz',
description: 'text',
location: 'text',
iCalUID: 'text',
conferenceSolution: 'text',
conferenceUri: 'text',
recurringEventExternalId: 'text',
});
await this.workspaceDataSourceService.executeRawQuery(
`UPDATE ${dataSourceSchema}."calendarEvent" AS "calendarEvent"
SET "title" = "newData"."title",
"isCanceled" = "newData"."isCanceled",
"isFullDay" = "newData"."isFullDay",
"startsAt" = "newData"."startsAt",
"endsAt" = "newData"."endsAt",
"externalCreatedAt" = "newData"."externalCreatedAt",
"externalUpdatedAt" = "newData"."externalUpdatedAt",
"description" = "newData"."description",
"location" = "newData"."location",
"conferenceSolution" = "newData"."conferenceSolution",
"conferenceUri" = "newData"."conferenceUri",
"recurringEventExternalId" = "newData"."recurringEventExternalId"
FROM (VALUES ${valuesString})
AS "newData"("title", "isCanceled", "isFullDay", "startsAt", "endsAt", "externalCreatedAt", "externalUpdatedAt", "description", "location", "iCalUID", "conferenceSolution", "conferenceUri", "recurringEventExternalId")
WHERE "calendarEvent"."iCalUID" = "newData"."iCalUID"`,
flattenedValues,
workspaceId,
transactionManager,
);
}
}

View File

@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { CalendarEventModule } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.module';
import { CalendarEventCleanerService } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service';
@Module({
imports: [DataSourceModule, TypeORMModule, CalendarEventModule],
providers: [CalendarEventCleanerService],
exports: [CalendarEventCleanerService],
})
export class CalendarEventCleanerModule {}

View File

@ -0,0 +1,20 @@
import { Injectable } from '@nestjs/common';
import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service';
import { deleteUsingPagination } from 'src/workspace/messaging/services/thread-cleaner/utils/delete-using-pagination.util';
@Injectable()
export class CalendarEventCleanerService {
constructor(private readonly calendarEventService: CalendarEventService) {}
public async cleanWorkspaceCalendarEvents(workspaceId: string) {
await deleteUsingPagination(
workspaceId,
500,
this.calendarEventService.getNonAssociatedCalendarEventIdsPaginated.bind(
this.calendarEventService,
),
this.calendarEventService.deleteByIds.bind(this.calendarEventService),
);
}
}

View File

@ -0,0 +1,274 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/core/feature-flag/feature-flag.entity';
import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider';
import { googleCalendarSearchFilterExcludeEmails } from 'src/workspace/calendar/utils/google-calendar-search-filter.util';
import { CalendarChannelEventAssociationService } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service';
import { CalendarChannelService } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.service';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service';
import { formatGoogleCalendarEvent } from 'src/workspace/calendar/utils/format-google-calendar-event.util';
import { GoogleCalendarFullSyncJobData } from 'src/workspace/calendar/jobs/google-calendar-full-sync.job';
import { CalendarEventAttendeeService } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service';
@Injectable()
export class GoogleCalendarFullSyncService {
private readonly logger = new Logger(GoogleCalendarFullSyncService.name);
constructor(
private readonly googleCalendarClientProvider: GoogleCalendarClientProvider,
@Inject(MessageQueue.calendarQueue)
private readonly messageQueueService: MessageQueueService,
private readonly connectedAccountService: ConnectedAccountService,
private readonly calendarEventService: CalendarEventService,
private readonly calendarChannelService: CalendarChannelService,
private readonly calendarChannelEventAssociationService: CalendarChannelEventAssociationService,
private readonly calendarEventAttendeesService: CalendarEventAttendeeService,
private readonly blocklistService: BlocklistService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async startGoogleCalendarFullSync(
workspaceId: string,
connectedAccountId: string,
pageToken?: string,
): Promise<void> {
const connectedAccount = await this.connectedAccountService.getById(
connectedAccountId,
workspaceId,
);
if (!connectedAccount) {
return;
}
const refreshToken = connectedAccount.refreshToken;
const workspaceMemberId = connectedAccount.accountOwnerId;
if (!refreshToken) {
throw new Error(
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
);
}
const calendarChannel =
await this.calendarChannelService.getFirstByConnectedAccountIdOrFail(
connectedAccountId,
workspaceId,
);
const calendarChannelId = calendarChannel.id;
const googleCalendarClient =
await this.googleCalendarClientProvider.getGoogleCalendarClient(
refreshToken,
);
const isBlocklistEnabledFeatureFlag =
await this.featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKeys.IsBlocklistEnabled,
value: true,
});
const isBlocklistEnabled =
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
const blocklist = isBlocklistEnabled
? await this.blocklistService.getByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
)
: [];
const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
let startTime = Date.now();
const googleCalendarEvents = await googleCalendarClient.events.list({
calendarId: 'primary',
maxResults: 500,
pageToken: pageToken,
q: googleCalendarSearchFilterExcludeEmails(blocklistedEmails),
});
let endTime = Date.now();
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} getting events list in ${
endTime - startTime
}ms.`,
);
const {
items: events,
nextPageToken,
nextSyncToken,
} = googleCalendarEvents.data;
if (!events || events?.length === 0) {
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
);
return;
}
const eventExternalIds = events.map((event) => event.id as string);
startTime = Date.now();
const existingCalendarChannelEventAssociations =
await this.calendarChannelEventAssociationService.getByEventExternalIdsAndCalendarChannelId(
eventExternalIds,
calendarChannelId,
workspaceId,
);
endTime = Date.now();
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: getting existing calendar channel event associations in ${
endTime - startTime
}ms.`,
);
// TODO: In V2, we will also import deleted events by doing batch GET queries on the canceled events
// The canceled events start and end are not accessible in the list query
const formattedEvents = events
.filter((event) => event.status !== 'cancelled')
.map((event) => formatGoogleCalendarEvent(event));
// TODO: When we will be able to add unicity contraint on iCalUID, we will do a INSERT ON CONFLICT DO UPDATE
const existingEventExternalIds =
existingCalendarChannelEventAssociations.map(
(association) => association.eventExternalId,
);
const eventsToSave = formattedEvents.filter(
(event) => !existingEventExternalIds.includes(event.externalId),
);
const eventsToUpdate = formattedEvents.filter((event) =>
existingEventExternalIds.includes(event.externalId),
);
const calendarChannelEventAssociationsToSave = eventsToSave.map(
(event) => ({
calendarEventId: event.id,
eventExternalId: event.externalId,
calendarChannelId,
}),
);
const attendeesToSave = eventsToSave.flatMap((event) => event.attendees);
const attendeesToUpdate = eventsToUpdate.flatMap(
(event) => event.attendees,
);
const iCalUIDCalendarEventIdMap =
await this.calendarEventService.getICalUIDCalendarEventIdMap(
eventsToUpdate.map((event) => event.iCalUID),
workspaceId,
);
if (events.length > 0) {
const dataSourceMetadata =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
dataSourceMetadata?.transaction(async (transactionManager) => {
this.calendarEventService.saveCalendarEvents(
eventsToSave,
workspaceId,
transactionManager,
);
this.calendarEventService.updateCalendarEvents(
eventsToUpdate,
workspaceId,
transactionManager,
);
this.calendarChannelEventAssociationService.saveCalendarChannelEventAssociations(
calendarChannelEventAssociationsToSave,
workspaceId,
transactionManager,
);
this.calendarEventAttendeesService.saveCalendarEventAttendees(
attendeesToSave,
workspaceId,
transactionManager,
);
this.calendarEventAttendeesService.updateCalendarEventAttendees(
attendeesToUpdate,
iCalUIDCalendarEventIdMap,
workspaceId,
transactionManager,
);
});
} else {
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
);
}
if (!nextSyncToken) {
throw new Error(
`No next sync token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
);
}
startTime = Date.now();
// await this.calendarChannelService.updateSyncCursor(
// nextSyncToken,
// connectedAccount.id,
// workspaceId,
// );
endTime = Date.now();
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating sync cursor in ${
endTime - startTime
}ms.`,
);
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} ${
nextPageToken ? `and ${nextPageToken} pageToken` : ''
} done.`,
);
if (nextPageToken) {
await this.messageQueueService.add<GoogleCalendarFullSyncJobData>(
GoogleCalendarFullSyncService.name,
{
workspaceId,
connectedAccountId,
nextPageToken,
},
{
retryLimit: 2,
},
);
}
}
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { EnvironmentModule } from 'src/integrations/environment/environment.module';
import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider';
@Module({
imports: [EnvironmentModule],
providers: [GoogleCalendarClientProvider],
exports: [GoogleCalendarClientProvider],
})
export class CalendarProvidersModule {}

View File

@ -0,0 +1,42 @@
import { Injectable } from '@nestjs/common';
import { OAuth2Client } from 'google-auth-library';
import { calendar_v3 as calendarV3, google } from 'googleapis';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@Injectable()
export class GoogleCalendarClientProvider {
constructor(private readonly environmentService: EnvironmentService) {}
public async getGoogleCalendarClient(
refreshToken: string,
): Promise<calendarV3.Calendar> {
const oAuth2Client = await this.getOAuth2Client(refreshToken);
const googleCalendarClient = google.calendar({
version: 'v3',
auth: oAuth2Client,
});
return googleCalendarClient;
}
private async getOAuth2Client(refreshToken: string): Promise<OAuth2Client> {
const googleCalendarClientId =
this.environmentService.getAuthGoogleClientId();
const googleCalendarClientSecret =
this.environmentService.getAuthGoogleClientSecret();
const oAuth2Client = new google.auth.OAuth2(
googleCalendarClientId,
googleCalendarClientSecret,
);
oAuth2Client.setCredentials({
refresh_token: refreshToken,
});
return oAuth2Client;
}
}

View File

@ -0,0 +1,31 @@
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
import { CalendarEventObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
export type CalendarEvent = Omit<
ObjectRecord<CalendarEventObjectMetadata>,
| 'createdAt'
| 'updatedAt'
| 'calendarChannelEventAssociations'
| 'calendarEventAttendees'
| 'eventAttendees'
>;
export type CalendarEventAttendee = Omit<
ObjectRecord<CalendarEventAttendeeObjectMetadata>,
| 'id'
| 'createdAt'
| 'updatedAt'
| 'personId'
| 'workspaceMemberId'
| 'person'
| 'workspaceMember'
| 'calendarEvent'
> & {
iCalUID: string;
};
export type CalendarEventWithAttendees = CalendarEvent & {
externalId: string;
attendees: CalendarEventAttendee[];
};

View File

@ -0,0 +1,52 @@
import { calendar_v3 } from 'googleapis';
import { v4 } from 'uuid';
import { CalendarEventWithAttendees } from 'src/workspace/calendar/types/calendar-event';
import { CalendarEventAttendeeResponseStatus } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
export const formatGoogleCalendarEvent = (
event: calendar_v3.Schema$Event,
): CalendarEventWithAttendees => {
const id = v4();
const formatResponseStatus = (status: string | null | undefined) => {
switch (status) {
case 'accepted':
return CalendarEventAttendeeResponseStatus.ACCEPTED;
case 'declined':
return CalendarEventAttendeeResponseStatus.DECLINED;
case 'tentative':
return CalendarEventAttendeeResponseStatus.TENTATIVE;
default:
return CalendarEventAttendeeResponseStatus.NEEDS_ACTION;
}
};
return {
id,
title: event.summary ?? '',
isCanceled: event.status === 'cancelled',
isFullDay: event.start?.dateTime == null,
startsAt: event.start?.dateTime ?? event.start?.date ?? null,
endsAt: event.end?.dateTime ?? event.end?.date ?? null,
externalId: event.id ?? '',
externalCreatedAt: event.created ?? null,
externalUpdatedAt: event.updated ?? null,
description: event.description ?? '',
location: event.location ?? '',
iCalUID: event.iCalUID ?? '',
conferenceSolution:
event.conferenceData?.conferenceSolution?.key?.type ?? '',
conferenceUri: event.conferenceData?.entryPoints?.[0]?.uri ?? '',
recurringEventExternalId: event.recurringEventId ?? '',
attendees:
event.attendees?.map((attendee) => ({
calendarEventId: id,
iCalUID: event.iCalUID ?? '',
handle: attendee.email ?? '',
displayName: attendee.displayName ?? '',
isOrganizer: attendee.organizer === true,
responseStatus: formatResponseStatus(attendee.responseStatus),
})) ?? [],
};
};

View File

@ -0,0 +1,9 @@
export const googleCalendarSearchFilterExcludeEmails = (
emails: string[],
): string => {
if (emails.length === 0) {
return '';
}
return `email=-${emails.join(', -')}`;
};

View File

@ -7,7 +7,7 @@ import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/messa
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { Workspace } from 'src/core/workspace/workspace.entity';
import {
GmailPartialSyncJobData,

View File

@ -6,7 +6,7 @@ import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { GmailFullSyncCommand } from 'src/workspace/messaging/commands/gmail-full-sync.command';
import { GmailPartialSyncCommand } from 'src/workspace/messaging/commands/gmail-partial-sync.command';
import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { StartFetchAllWorkspacesMessagesCronCommand } from 'src/workspace/messaging/commands/start-fetch-all-workspaces-messages.cron.command';
import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/workspace/messaging/commands/stop-fetch-all-workspaces-messages.cron.command';

View File

@ -8,7 +8,7 @@ import {
GmailFullSyncJobData,
GmailFullSyncJob,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
interface GmailFullSyncOptions {
workspaceId: string;

View File

@ -8,7 +8,7 @@ import {
GmailPartialSyncJob,
GmailPartialSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-partial-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
interface GmailPartialSyncOptions {
workspaceId: string;

View File

@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service';
import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service';
@ -19,7 +19,7 @@ export class CreateCompaniesAndContactsAfterSyncJob
CreateCompaniesAndContactsAfterSyncJob.name,
);
constructor(
private readonly createCompaniesAndContactsService: CreateCompaniesAndContactsService,
private readonly createCompaniesAndContactsService: CreateCompanyAndContactService,
private readonly messageChannelService: MessageChannelService,
private readonly messageParticipantService: MessageParticipantService,
) {}

View File

@ -0,0 +1,40 @@
import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { CalendarEventCleanerService } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service';
export type DeleteConnectedAccountAssociatedCalendarDataJobData = {
workspaceId: string;
connectedAccountId: string;
};
@Injectable()
export class DeleteConnectedAccountAssociatedCalendarDataJob
implements
MessageQueueJob<DeleteConnectedAccountAssociatedCalendarDataJobData>
{
private readonly logger = new Logger(
DeleteConnectedAccountAssociatedCalendarDataJob.name,
);
constructor(
private readonly calendarEventCleanerService: CalendarEventCleanerService,
) {}
async handle(
data: DeleteConnectedAccountAssociatedCalendarDataJobData,
): Promise<void> {
this.logger.log(
`Deleting connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`,
);
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
data.workspaceId,
);
this.logger.log(
`Deleted connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`,
);
}
}

View File

@ -4,32 +4,33 @@ import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/messa
import { ThreadCleanerService } from 'src/workspace/messaging/services/thread-cleaner/thread-cleaner.service';
export type DeleteConnectedAccountAssociatedDataJobData = {
export type DeleteConnectedAccountAssociatedMessagingDataJobData = {
workspaceId: string;
connectedAccountId: string;
};
@Injectable()
export class DeleteConnectedAccountAssociatedDataJob
implements MessageQueueJob<DeleteConnectedAccountAssociatedDataJobData>
export class DeleteConnectedAccountAssociatedMessagingDataJob
implements
MessageQueueJob<DeleteConnectedAccountAssociatedMessagingDataJobData>
{
private readonly logger = new Logger(
DeleteConnectedAccountAssociatedDataJob.name,
DeleteConnectedAccountAssociatedMessagingDataJob.name,
);
constructor(private readonly threadCleanerService: ThreadCleanerService) {}
async handle(
data: DeleteConnectedAccountAssociatedDataJobData,
data: DeleteConnectedAccountAssociatedMessagingDataJobData,
): Promise<void> {
this.logger.log(
`Deleting connected account ${data.connectedAccountId} associated data in workspace ${data.workspaceId}`,
`Deleting connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`,
);
await this.threadCleanerService.cleanWorkspaceThreads(data.workspaceId);
this.logger.log(
`Deleted connected account ${data.connectedAccountId} associated data in workspace ${data.workspaceId}`,
`Deleted connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`,
);
}
}

View File

@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service';
export type GmailFullSyncJobData = {
@ -16,7 +16,7 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
private readonly logger = new Logger(GmailFullSyncJob.name);
constructor(
private readonly gmailRefreshAccessTokenService: GmailRefreshAccessTokenService,
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService,
private readonly gmailFullSyncService: GmailFullSyncService,
) {}
@ -28,7 +28,7 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
);
try {
await this.gmailRefreshAccessTokenService.refreshAndSaveAccessToken(
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
data.workspaceId,
data.connectedAccountId,
);

View File

@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { GmailPartialSyncService } from 'src/workspace/messaging/services/gmail-partial-sync.service';
export type GmailPartialSyncJobData = {
@ -17,7 +17,7 @@ export class GmailPartialSyncJob
private readonly logger = new Logger(GmailPartialSyncJob.name);
constructor(
private readonly gmailRefreshAccessTokenService: GmailRefreshAccessTokenService,
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService,
private readonly gmailPartialSyncService: GmailPartialSyncService,
) {}
@ -27,7 +27,7 @@ export class GmailPartialSyncJob
);
try {
await this.gmailRefreshAccessTokenService.refreshAndSaveAccessToken(
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
data.workspaceId,
data.connectedAccountId,
);

View File

@ -5,9 +5,13 @@ import { ObjectRecordDeleteEvent } from 'src/integrations/event-emitter/types/ob
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import {
DeleteConnectedAccountAssociatedDataJobData,
DeleteConnectedAccountAssociatedDataJob,
} from 'src/workspace/messaging/jobs/delete-connected-acount-associated-data.job';
DeleteConnectedAccountAssociatedCalendarDataJobData,
DeleteConnectedAccountAssociatedCalendarDataJob,
} from 'src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job';
import {
DeleteConnectedAccountAssociatedMessagingDataJobData,
DeleteConnectedAccountAssociatedMessagingDataJob,
} from 'src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job';
import { ConnectedAccountObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/connected-account.object-metadata';
@Injectable()
@ -15,14 +19,24 @@ export class MessagingConnectedAccountListener {
constructor(
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@Inject(MessageQueue.calendarQueue)
private readonly calendarQueueService: MessageQueueService,
) {}
@OnEvent('connectedAccount.deleted')
handleDeletedEvent(
payload: ObjectRecordDeleteEvent<ConnectedAccountObjectMetadata>,
) {
this.messageQueueService.add<DeleteConnectedAccountAssociatedDataJobData>(
DeleteConnectedAccountAssociatedDataJob.name,
this.messageQueueService.add<DeleteConnectedAccountAssociatedMessagingDataJobData>(
DeleteConnectedAccountAssociatedMessagingDataJob.name,
{
workspaceId: payload.workspaceId,
connectedAccountId: payload.deletedRecord.id,
},
);
this.calendarQueueService.add<DeleteConnectedAccountAssociatedCalendarDataJobData>(
DeleteConnectedAccountAssociatedCalendarDataJob.name,
{
workspaceId: payload.workspaceId,
connectedAccountId: payload.deletedRecord.id,

View File

@ -1,7 +1,8 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { HttpModule } from '@nestjs/axios';
import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { MessageChannelMessageAssociationModule } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-assocation.module';
import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module';
import { MessageThreadModule } from 'src/workspace/messaging/repositories/message-thread/message-thread.module';
@ -9,25 +10,26 @@ import { EnvironmentModule } from 'src/integrations/environment/environment.modu
import { MessagingPersonListener } from 'src/workspace/messaging/listeners/messaging-person.listener';
import { MessageModule } from 'src/workspace/messaging/repositories/message/message.module';
import { GmailClientProvider } from 'src/workspace/messaging/services/providers/gmail/gmail-client.provider';
import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service';
import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service';
import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service';
import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service';
import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service';
import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service';
import { GmailPartialSyncService } from 'src/workspace/messaging/services/gmail-partial-sync.service';
import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { MessageParticipantModule } from 'src/workspace/messaging/repositories/message-participant/message-participant.module';
import { MessagingWorkspaceMemberListener } from 'src/workspace/messaging/listeners/messaging-workspace-member.listener';
import { MessagingMessageChannelListener } from 'src/workspace/messaging/listeners/messaging-message-channel.listener';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
import { MessagingConnectedAccountListener } from 'src/workspace/messaging/listeners/messaging-connected-account.listener';
import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/blocklist.module';
import { BlocklistModule } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module';
import { FetchByBatchesService } from 'src/workspace/messaging/services/fetch-by-batch.service';
@Module({
imports: [
EnvironmentModule,
@ -44,12 +46,15 @@ import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/
CompanyModule,
PersonModule,
BlocklistModule,
HttpModule.register({
baseURL: 'https://www.googleapis.com/batch/gmail/v1',
}),
],
providers: [
GmailFullSyncService,
GmailPartialSyncService,
FetchMessagesByBatchesService,
GmailRefreshAccessTokenService,
GoogleAPIsRefreshAccessTokenService,
GmailClientProvider,
CreateContactService,
CreateCompanyService,
@ -59,11 +64,13 @@ import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/
MessageService,
SaveMessagesAndCreateContactsService,
MessagingConnectedAccountListener,
FetchByBatchesService,
],
exports: [
GmailPartialSyncService,
GmailFullSyncService,
GmailRefreshAccessTokenService,
GoogleAPIsRefreshAccessTokenService,
FetchByBatchesService,
],
})
export class MessagingModule {}

View File

@ -12,8 +12,8 @@ import { FindManyResolverArgs } from 'src/workspace/workspace-resolver-builder/i
import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-association.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service';
@Injectable()
export class MessageFindManyPreQueryHook implements WorkspacePreQueryHook {

View File

@ -2,10 +2,10 @@ import { Module } from '@nestjs/common';
import { MessageFindManyPreQueryHook } from 'src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook';
import { MessageFindOnePreQueryHook } from 'src/workspace/messaging/query-hooks/message/message-find-one.pre-query-hook';
import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { MessageChannelMessageAssociationModule } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-assocation.module';
import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module';
import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
@Module({
imports: [

View File

@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
@Module({
imports: [WorkspaceDataSourceModule, PersonModule],

View File

@ -9,7 +9,7 @@ import {
ParticipantWithId,
ParticipantWithMessageId,
} from 'src/workspace/messaging/types/gmail-message';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
@Injectable()
export class MessageParticipantService {

View File

@ -6,7 +6,7 @@ import { MessageParticipantModule } from 'src/workspace/messaging/repositories/m
import { MessageThreadModule } from 'src/workspace/messaging/repositories/message-thread/message-thread.module';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
@Module({
imports: [

View File

@ -1,21 +0,0 @@
import { Module } from '@nestjs/common';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module';
import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service';
import { CreateCompanyModule } from 'src/workspace/messaging/services/create-company/create-company.module';
import { CreateContactModule } from 'src/workspace/messaging/services/create-contact/create-contact.module';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
WorkspaceDataSourceModule,
CreateContactModule,
CreateCompanyModule,
WorkspaceMemberModule,
PersonModule,
],
providers: [CreateCompaniesAndContactsService],
exports: [CreateCompaniesAndContactsService],
})
export class CreateCompaniesAndContactsModule {}

View File

@ -0,0 +1,128 @@
import { HttpService } from '@nestjs/axios';
import { Injectable } from '@nestjs/common';
import { AxiosResponse } from 'axios';
import { BatchQueries } from 'src/workspace/calendar-and-messaging/types/batch-queries';
import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmail-message-parsed-response';
@Injectable()
export class FetchByBatchesService {
constructor(private readonly httpService: HttpService) {}
async fetchAllByBatches(
queries: BatchQueries,
accessToken: string,
boundary: string,
): Promise<AxiosResponse<any, any>[]> {
const batchLimit = 50;
let batchOffset = 0;
let batchResponses: AxiosResponse<any, any>[] = [];
while (batchOffset < queries.length) {
const batchResponse = await this.fetchBatch(
queries,
accessToken,
batchOffset,
batchLimit,
boundary,
);
batchResponses = batchResponses.concat(batchResponse);
batchOffset += batchLimit;
}
return batchResponses;
}
async fetchBatch(
queries: BatchQueries,
accessToken: string,
batchOffset: number,
batchLimit: number,
boundary: string,
): Promise<AxiosResponse<any, any>> {
const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit);
const response = await this.httpService.axiosRef.post(
'/',
this.createBatchBody(limitedQueries, boundary),
{
headers: {
'Content-Type': 'multipart/mixed; boundary=' + boundary,
Authorization: 'Bearer ' + accessToken,
},
},
);
return response;
}
createBatchBody(queries: BatchQueries, boundary: string): string {
let batchBody: string[] = [];
queries.forEach(function (call) {
const method = 'GET';
const uri = call.uri;
batchBody = batchBody.concat([
'--',
boundary,
'\r\n',
'Content-Type: application/http',
'\r\n\r\n',
method,
' ',
uri,
'\r\n\r\n',
]);
});
return batchBody.concat(['--', boundary, '--']).join('');
}
parseBatch(
responseCollection: AxiosResponse<any, any>,
): GmailMessageParsedResponse[] {
const responseItems: GmailMessageParsedResponse[] = [];
const boundary = this.getBatchSeparator(responseCollection);
const responseLines: string[] = responseCollection.data.split(
'--' + boundary,
);
responseLines.forEach(function (response) {
const startJson = response.indexOf('{');
const endJson = response.lastIndexOf('}');
if (startJson < 0 || endJson < 0) return;
const responseJson = response.substring(startJson, endJson + 1);
const item = JSON.parse(responseJson);
responseItems.push(item);
});
return responseItems;
}
getBatchSeparator(responseCollection: AxiosResponse<any, any>): string {
const headers = responseCollection.headers;
const contentType: string = headers['content-type'];
if (!contentType) return '';
const components = contentType.split('; ');
const boundary = components.find((item) => item.startsWith('boundary='));
return boundary?.replace('boundary=', '').trim() || '';
}
}

View File

@ -1,6 +1,6 @@
import { Injectable, Logger } from '@nestjs/common';
import axios, { AxiosInstance, AxiosResponse } from 'axios';
import { AxiosResponse } from 'axios';
import { simpleParser, AddressObject } from 'mailparser';
import planer from 'planer';
@ -10,17 +10,13 @@ import {
} from 'src/workspace/messaging/types/gmail-message';
import { MessageQuery } from 'src/workspace/messaging/types/message-or-thread-query';
import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmail-message-parsed-response';
import { FetchByBatchesService } from 'src/workspace/messaging/services/fetch-by-batch.service';
@Injectable()
export class FetchMessagesByBatchesService {
private readonly httpService: AxiosInstance;
private readonly logger = new Logger(FetchMessagesByBatchesService.name);
constructor() {
this.httpService = axios.create({
baseURL: 'https://www.googleapis.com/batch/gmail/v1',
});
}
constructor(private readonly fetchByBatchesService: FetchByBatchesService) {}
async fetchAllMessages(
queries: MessageQuery[],
@ -30,7 +26,7 @@ export class FetchMessagesByBatchesService {
connectedAccountId?: string,
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
let startTime = Date.now();
const batchResponses = await this.fetchAllByBatches(
const batchResponses = await this.fetchByBatchesService.fetchAllByBatches(
queries,
accessToken,
'batch_gmail_messages',
@ -59,126 +55,10 @@ export class FetchMessagesByBatchesService {
return formattedResponse;
}
async fetchAllByBatches(
queries: MessageQuery[],
accessToken: string,
boundary: string,
): Promise<AxiosResponse<any, any>[]> {
const batchLimit = 50;
let batchOffset = 0;
let batchResponses: AxiosResponse<any, any>[] = [];
while (batchOffset < queries.length) {
const batchResponse = await this.fetchBatch(
queries,
accessToken,
batchOffset,
batchLimit,
boundary,
);
batchResponses = batchResponses.concat(batchResponse);
batchOffset += batchLimit;
}
return batchResponses;
}
async fetchBatch(
queries: MessageQuery[],
accessToken: string,
batchOffset: number,
batchLimit: number,
boundary: string,
): Promise<AxiosResponse<any, any>> {
const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit);
const response = await this.httpService.post(
'/',
this.createBatchBody(limitedQueries, boundary),
{
headers: {
'Content-Type': 'multipart/mixed; boundary=' + boundary,
Authorization: 'Bearer ' + accessToken,
},
},
);
return response;
}
createBatchBody(queries: MessageQuery[], boundary: string): string {
let batchBody: string[] = [];
queries.forEach(function (call) {
const method = 'GET';
const uri = call.uri;
batchBody = batchBody.concat([
'--',
boundary,
'\r\n',
'Content-Type: application/http',
'\r\n\r\n',
method,
' ',
uri,
'\r\n\r\n',
]);
});
return batchBody.concat(['--', boundary, '--']).join('');
}
parseBatch(
responseCollection: AxiosResponse<any, any>,
): GmailMessageParsedResponse[] {
const responseItems: GmailMessageParsedResponse[] = [];
const boundary = this.getBatchSeparator(responseCollection);
const responseLines: string[] = responseCollection.data.split(
'--' + boundary,
);
responseLines.forEach(function (response) {
const startJson = response.indexOf('{');
const endJson = response.lastIndexOf('}');
if (startJson < 0 || endJson < 0) return;
const responseJson = response.substring(startJson, endJson + 1);
const item = JSON.parse(responseJson);
responseItems.push(item);
});
return responseItems;
}
getBatchSeparator(responseCollection: AxiosResponse<any, any>): string {
const headers = responseCollection.headers;
const contentType: string = headers['content-type'];
if (!contentType) return '';
const components = contentType.split('; ');
const boundary = components.find((item) => item.startsWith('boundary='));
return boundary?.replace('boundary=', '').trim() || '';
}
async formatBatchResponseAsGmailMessage(
responseCollection: AxiosResponse<any, any>,
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
const parsedResponses = this.parseBatch(
const parsedResponses = this.fetchByBatchesService.parseBatch(
responseCollection,
) as GmailMessageParsedResponse[];

View File

@ -11,12 +11,12 @@ import {
GmailFullSyncJobData,
GmailFullSyncJob,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-association.service';
import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
import { gmailSearchFilterExcludeEmails } from 'src/workspace/messaging/utils/gmail-search-filter';
import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
import { gmailSearchFilterExcludeEmails } from 'src/workspace/messaging/utils/gmail-search-filter.util';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
import {
FeatureFlagEntity,

View File

@ -12,13 +12,13 @@ import {
GmailFullSyncJob,
GmailFullSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
import { GmailMessage } from 'src/workspace/messaging/types/gmail-message';
import { isPersonEmail } from 'src/workspace/messaging/utils/is-person-email.util';
import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
import {
FeatureFlagEntity,

View File

@ -5,7 +5,7 @@ import { EntityManager } from 'typeorm';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service';
import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service';
import {
GmailMessage,
ParticipantWithMessageId,
@ -23,7 +23,7 @@ export class SaveMessagesAndCreateContactsService {
constructor(
private readonly messageService: MessageService,
private readonly messageChannelService: MessageChannelService,
private readonly createCompaniesAndContactsService: CreateCompaniesAndContactsService,
private readonly createCompaniesAndContactsService: CreateCompanyAndContactService,
private readonly messageParticipantService: MessageParticipantService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}

View File

@ -1,9 +1,8 @@
import { Module } from '@nestjs/common';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
// TODO: Move outside of the messaging module
@Module({
imports: [WorkspaceDataSourceModule],
providers: [PersonService],

View File

@ -1,9 +1,8 @@
import { Module } from '@nestjs/common';
import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service';
import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
// TODO: Move outside of the messaging module
@Module({
imports: [WorkspaceDataSourceModule],
providers: [WorkspaceMemberService],

View File

@ -12,6 +12,7 @@ import { ObjectMetadata } from 'src/workspace/workspace-sync-metadata/decorators
import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
import { CalendarChannelEventAssociationObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel-event-association.object-metadata';
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
import { IsNullable } from 'src/workspace/workspace-sync-metadata/decorators/is-nullable.decorator';
import { standardObjectIds } from 'src/workspace/workspace-sync-metadata/constants/standard-object-ids';
import { calendarEventStandardFieldIds } from 'src/workspace/workspace-sync-metadata/constants/standard-field-ids';
@ -62,7 +63,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'Start DateTime',
icon: 'IconCalendarClock',
})
startsAt: string;
@IsNullable()
startsAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.endsAt,
@ -71,7 +73,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'End DateTime',
icon: 'IconCalendarClock',
})
endsAt: string;
@IsNullable()
endsAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.externalCreatedAt,
@ -80,7 +83,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'Creation DateTime',
icon: 'IconCalendarPlus',
})
externalCreatedAt: string;
@IsNullable()
externalCreatedAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.externalUpdatedAt,
@ -89,7 +93,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'Update DateTime',
icon: 'IconCalendarCog',
})
externalUpdatedAt: string;
@IsNullable()
externalUpdatedAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.description,

View File

@ -6,6 +6,7 @@ import { WorkspaceSchemaStorageModule } from 'src/workspace/workspace-schema-sto
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { ScalarsExplorerService } from 'src/workspace/services/scalars-explorer.service';
import { MessagingModule } from 'src/workspace/messaging/messaging.module';
import { CalendarModule } from 'src/workspace/calendar/calendar.module';
import { WorkspaceFactory } from './workspace.factory';
@ -21,6 +22,7 @@ import { WorkspaceResolverBuilderModule } from './workspace-resolver-builder/wor
WorkspaceResolverBuilderModule,
WorkspaceSchemaStorageModule,
MessagingModule,
CalendarModule,
],
providers: [WorkspaceFactory, ScalarsExplorerService],
exports: [WorkspaceFactory],