6657 Refactor and fix blocklist (#6803)

Closes #6657
- Fix listeners
- Refactor jobs to take array of events
- Fix calendar events and messages deletion

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Raphaël Bosi
2024-08-31 16:38:47 +02:00
committed by GitHub
parent d9650fd5cf
commit cd66ea74a2
37 changed files with 799 additions and 699 deletions

View File

@ -1,5 +1,4 @@
export type FeatureFlagKey = export type FeatureFlagKey =
| 'IS_BLOCKLIST_ENABLED'
| 'IS_EVENT_OBJECT_ENABLED' | 'IS_EVENT_OBJECT_ENABLED'
| 'IS_AIRTABLE_INTEGRATION_ENABLED' | 'IS_AIRTABLE_INTEGRATION_ENABLED'
| 'IS_POSTGRESQL_INTEGRATION_ENABLED' | 'IS_POSTGRESQL_INTEGRATION_ENABLED'

View File

@ -32,9 +32,7 @@ export const Default: Story = {
play: async ({ canvasElement }) => { play: async ({ canvasElement }) => {
const canvas = within(canvasElement); const canvas = within(canvasElement);
await canvas.findByText('People'); await canvas.findByText('People', undefined, { timeout: 3000 });
await canvas.findAllByText('Companies'); await canvas.findByText('Linkedin');
await canvas.findByText('Opportunities');
await canvas.findByText('My Customs');
}, },
}; };

View File

@ -14,7 +14,6 @@ import { SettingsAccountsSettingsSection } from '@/settings/accounts/components/
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer'; import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
import { SubMenuTopBarContainer } from '@/ui/layout/page/SubMenuTopBarContainer'; import { SubMenuTopBarContainer } from '@/ui/layout/page/SubMenuTopBarContainer';
import { Section } from '@/ui/layout/section/components/Section'; import { Section } from '@/ui/layout/section/components/Section';
import { useIsFeatureEnabled } from '@/workspace/hooks/useIsFeatureEnabled';
export const SettingsAccounts = () => { export const SettingsAccounts = () => {
const currentWorkspaceMember = useRecoilValue(currentWorkspaceMemberState); const currentWorkspaceMember = useRecoilValue(currentWorkspaceMemberState);
@ -33,8 +32,6 @@ export const SettingsAccounts = () => {
recordGqlFields: generateDepthOneRecordGqlFields({ objectMetadataItem }), recordGqlFields: generateDepthOneRecordGqlFields({ objectMetadataItem }),
}); });
const isBlocklistEnabled = useIsFeatureEnabled('IS_BLOCKLIST_ENABLED');
return ( return (
<SubMenuTopBarContainer Icon={IconAt} title="Account"> <SubMenuTopBarContainer Icon={IconAt} title="Account">
<SettingsPageContainer> <SettingsPageContainer>
@ -52,7 +49,7 @@ export const SettingsAccounts = () => {
loading={loading} loading={loading}
/> />
</Section> </Section>
{isBlocklistEnabled && <SettingsAccountsBlocklistSection />} <SettingsAccountsBlocklistSection />
<SettingsAccountsSettingsSection /> <SettingsAccountsSettingsSection />
</> </>
)} )}

View File

@ -15,11 +15,6 @@ export const seedFeatureFlags = async (
.into(`${schemaName}.${tableName}`, ['key', 'workspaceId', 'value']) .into(`${schemaName}.${tableName}`, ['key', 'workspaceId', 'value'])
.orIgnore() .orIgnore()
.values([ .values([
{
key: FeatureFlagKey.IsBlocklistEnabled,
workspaceId: workspaceId,
value: true,
},
{ {
key: FeatureFlagKey.IsAirtableIntegrationEnabled, key: FeatureFlagKey.IsAirtableIntegrationEnabled,
workspaceId: workspaceId, workspaceId: workspaceId,
@ -40,16 +35,6 @@ export const seedFeatureFlags = async (
workspaceId: workspaceId, workspaceId: workspaceId,
value: true, value: true,
}, },
{
key: FeatureFlagKey.IsMessagingAliasFetchingEnabled,
workspaceId: workspaceId,
value: true,
},
{
key: FeatureFlagKey.IsGoogleCalendarSyncV2Enabled,
workspaceId: workspaceId,
value: true,
},
{ {
key: FeatureFlagKey.IsFunctionSettingsEnabled, key: FeatureFlagKey.IsFunctionSettingsEnabled,
workspaceId: workspaceId, workspaceId: workspaceId,

View File

@ -50,16 +50,22 @@ export class EntityEventsToDbListener {
} }
private async handle(payload: WorkspaceEventBatch<ObjectRecordBaseEvent>) { private async handle(payload: WorkspaceEventBatch<ObjectRecordBaseEvent>) {
payload.events = payload.events.filter( const filteredEvents = payload.events.filter(
(event) => event.objectMetadata?.isAuditLogged, (event) => event.objectMetadata?.isAuditLogged,
); );
await this.messageQueueService.add< await this.messageQueueService.add<
WorkspaceEventBatch<ObjectRecordBaseEvent> WorkspaceEventBatch<ObjectRecordBaseEvent>
>(CreateAuditLogFromInternalEvent.name, payload); >(CreateAuditLogFromInternalEvent.name, {
...payload,
events: filteredEvents,
});
await this.messageQueueService.add< await this.messageQueueService.add<
WorkspaceEventBatch<ObjectRecordBaseEvent> WorkspaceEventBatch<ObjectRecordBaseEvent>
>(UpsertTimelineActivityFromInternalEvent.name, payload); >(UpsertTimelineActivityFromInternalEvent.name, {
...payload,
events: filteredEvents,
});
} }
} }

View File

@ -1,33 +1,19 @@
import { ExecutionContext, Injectable } from '@nestjs/common'; import { ExecutionContext, Injectable } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport'; import { AuthGuard } from '@nestjs/passport';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { import {
AuthException, AuthException,
AuthExceptionCode, AuthExceptionCode,
} from 'src/engine/core-modules/auth/auth.exception'; } from 'src/engine/core-modules/auth/auth.exception';
import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; import { GoogleAPIsOauthExchangeCodeForTokenStrategy } from 'src/engine/core-modules/auth/strategies/google-apis-oauth-exchange-code-for-token.auth.strategy';
import {
GoogleAPIScopeConfig,
GoogleAPIsOauthExchangeCodeForTokenStrategy,
} from 'src/engine/core-modules/auth/strategies/google-apis-oauth-exchange-code-for-token.auth.strategy';
import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util'; import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
@Injectable() @Injectable()
export class GoogleAPIsOauthExchangeCodeForTokenGuard extends AuthGuard( export class GoogleAPIsOauthExchangeCodeForTokenGuard extends AuthGuard(
'google-apis', 'google-apis',
) { ) {
constructor( constructor(private readonly environmentService: EnvironmentService) {
private readonly environmentService: EnvironmentService,
private readonly tokenService: TokenService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
) {
super(); super();
} }
@ -45,22 +31,9 @@ export class GoogleAPIsOauthExchangeCodeForTokenGuard extends AuthGuard(
); );
} }
const { workspaceId } = await this.tokenService.verifyTransientToken(
state.transientToken,
);
const scopeConfig: GoogleAPIScopeConfig = {
isMessagingAliasFetchingEnabled:
!!(await this.featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKey.IsMessagingAliasFetchingEnabled,
value: true,
})),
};
new GoogleAPIsOauthExchangeCodeForTokenStrategy( new GoogleAPIsOauthExchangeCodeForTokenStrategy(
this.environmentService, this.environmentService,
scopeConfig, {},
); );
setRequestExtraParams(request, { setRequestExtraParams(request, {

View File

@ -1,29 +1,17 @@
import { ExecutionContext, Injectable } from '@nestjs/common'; import { ExecutionContext, Injectable } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport'; import { AuthGuard } from '@nestjs/passport';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { import {
AuthException, AuthException,
AuthExceptionCode, AuthExceptionCode,
} from 'src/engine/core-modules/auth/auth.exception'; } from 'src/engine/core-modules/auth/auth.exception';
import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
import { GoogleAPIScopeConfig } from 'src/engine/core-modules/auth/strategies/google-apis-oauth-exchange-code-for-token.auth.strategy';
import { GoogleAPIsOauthRequestCodeStrategy } from 'src/engine/core-modules/auth/strategies/google-apis-oauth-request-code.auth.strategy'; import { GoogleAPIsOauthRequestCodeStrategy } from 'src/engine/core-modules/auth/strategies/google-apis-oauth-request-code.auth.strategy';
import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util'; import { setRequestExtraParams } from 'src/engine/core-modules/auth/utils/google-apis-set-request-extra-params.util';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
@Injectable() @Injectable()
export class GoogleAPIsOauthRequestCodeGuard extends AuthGuard('google-apis') { export class GoogleAPIsOauthRequestCodeGuard extends AuthGuard('google-apis') {
constructor( constructor(private readonly environmentService: EnvironmentService) {
private readonly environmentService: EnvironmentService,
private readonly tokenService: TokenService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
) {
super({ super({
prompt: 'select_account', prompt: 'select_account',
}); });
@ -42,23 +30,7 @@ export class GoogleAPIsOauthRequestCodeGuard extends AuthGuard('google-apis') {
); );
} }
const { workspaceId } = await this.tokenService.verifyTransientToken( new GoogleAPIsOauthRequestCodeStrategy(this.environmentService, {});
request.query.transientToken,
);
const scopeConfig: GoogleAPIScopeConfig = {
isMessagingAliasFetchingEnabled:
!!(await this.featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKey.IsMessagingAliasFetchingEnabled,
value: true,
})),
};
new GoogleAPIsOauthRequestCodeStrategy(
this.environmentService,
scopeConfig,
);
setRequestExtraParams(request, { setRequestExtraParams(request, {
transientToken: request.query.transientToken, transientToken: request.query.transientToken,
redirectLocation: request.query.redirectLocation, redirectLocation: request.query.redirectLocation,

View File

@ -1,5 +1,5 @@
import { PassportStrategy } from '@nestjs/passport';
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { PassportStrategy } from '@nestjs/passport';
import { Strategy } from 'passport-google-oauth20'; import { Strategy } from 'passport-google-oauth20';
@ -24,12 +24,9 @@ export class GoogleAPIsOauthCommonStrategy extends PassportStrategy(
'profile', 'profile',
'https://www.googleapis.com/auth/gmail.readonly', 'https://www.googleapis.com/auth/gmail.readonly',
'https://www.googleapis.com/auth/calendar.events', 'https://www.googleapis.com/auth/calendar.events',
'https://www.googleapis.com/auth/profile.emails.read',
]; ];
if (scopeConfig?.isMessagingAliasFetchingEnabled) {
scopes.push('https://www.googleapis.com/auth/profile.emails.read');
}
super({ super({
clientID: environmentService.get('AUTH_GOOGLE_CLIENT_ID'), clientID: environmentService.get('AUTH_GOOGLE_CLIENT_ID'),
clientSecret: environmentService.get('AUTH_GOOGLE_CLIENT_SECRET'), clientSecret: environmentService.get('AUTH_GOOGLE_CLIENT_SECRET'),

View File

@ -3,12 +3,11 @@ import { Injectable } from '@nestjs/common';
import { VerifyCallback } from 'passport-google-oauth20'; import { VerifyCallback } from 'passport-google-oauth20';
import { GoogleAPIsOauthCommonStrategy } from 'src/engine/core-modules/auth/strategies/google-apis-oauth-common.auth.strategy'; import { GoogleAPIsOauthCommonStrategy } from 'src/engine/core-modules/auth/strategies/google-apis-oauth-common.auth.strategy';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { GoogleAPIsRequest } from 'src/engine/core-modules/auth/types/google-api-request.type'; import { GoogleAPIsRequest } from 'src/engine/core-modules/auth/types/google-api-request.type';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
export type GoogleAPIScopeConfig = { export type GoogleAPIScopeConfig = {
isCalendarEnabled?: boolean; isCalendarEnabled?: boolean;
isMessagingAliasFetchingEnabled?: boolean;
}; };
@Injectable() @Injectable()

View File

@ -1,12 +1,9 @@
export enum FeatureFlagKey { export enum FeatureFlagKey {
IsBlocklistEnabled = 'IS_BLOCKLIST_ENABLED',
IsEventObjectEnabled = 'IS_EVENT_OBJECT_ENABLED', IsEventObjectEnabled = 'IS_EVENT_OBJECT_ENABLED',
IsAirtableIntegrationEnabled = 'IS_AIRTABLE_INTEGRATION_ENABLED', IsAirtableIntegrationEnabled = 'IS_AIRTABLE_INTEGRATION_ENABLED',
IsPostgreSQLIntegrationEnabled = 'IS_POSTGRESQL_INTEGRATION_ENABLED', IsPostgreSQLIntegrationEnabled = 'IS_POSTGRESQL_INTEGRATION_ENABLED',
IsStripeIntegrationEnabled = 'IS_STRIPE_INTEGRATION_ENABLED', IsStripeIntegrationEnabled = 'IS_STRIPE_INTEGRATION_ENABLED',
IsCopilotEnabled = 'IS_COPILOT_ENABLED', IsCopilotEnabled = 'IS_COPILOT_ENABLED',
IsMessagingAliasFetchingEnabled = 'IS_MESSAGING_ALIAS_FETCHING_ENABLED',
IsGoogleCalendarSyncV2Enabled = 'IS_GOOGLE_CALENDAR_SYNC_V2_ENABLED',
IsFreeAccessEnabled = 'IS_FREE_ACCESS_ENABLED', IsFreeAccessEnabled = 'IS_FREE_ACCESS_ENABLED',
IsFunctionSettingsEnabled = 'IS_FUNCTION_SETTINGS_ENABLED', IsFunctionSettingsEnabled = 'IS_FUNCTION_SETTINGS_ENABLED',
IsWorkflowEnabled = 'IS_WORKFLOW_ENABLED', IsWorkflowEnabled = 'IS_WORKFLOW_ENABLED',

View File

@ -2,67 +2,67 @@ import { isEmailBlocklisted } from 'src/modules/blocklist/utils/is-email-blockli
describe('isEmailBlocklisted', () => { describe('isEmailBlocklisted', () => {
it('should return true if email is blocklisted', () => { it('should return true if email is blocklisted', () => {
const channelHandle = 'abc@example.com'; const channelHandles = ['abc@example.com'];
const email = 'hello@twenty.com'; const email = 'hello@twenty.com';
const blocklist = ['hello@twenty.com', 'hey@twenty.com']; const blocklist = ['hello@twenty.com', 'hey@twenty.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(true); expect(result).toBe(true);
}); });
it('should return false if email is not blocklisted', () => { it('should return false if email is not blocklisted', () => {
const channelHandle = 'abc@example.com'; const channelHandles = ['abc@example.com'];
const email = 'hello@twenty.com'; const email = 'hello@twenty.com';
const blocklist = ['hey@example.com']; const blocklist = ['hey@example.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(false); expect(result).toBe(false);
}); });
it('should return false if email is null', () => { it('should return false if email is null', () => {
const channelHandle = 'abc@twenty.com'; const channelHandles = ['abc@twenty.com'];
const email = null; const email = null;
const blocklist = ['@example.com']; const blocklist = ['@example.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(false); expect(result).toBe(false);
}); });
it('should return true for subdomains', () => { it('should return true for subdomains', () => {
const channelHandle = 'abc@example.com'; const channelHandles = ['abc@example.com'];
const email = 'hello@twenty.twenty.com'; const email = 'hello@twenty.twenty.com';
const blocklist = ['@twenty.com']; const blocklist = ['@twenty.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(true); expect(result).toBe(true);
}); });
it('should return false for domains which end with blocklisted domain but are not subdomains', () => { it('should return false for domains which end with blocklisted domain but are not subdomains', () => {
const channelHandle = 'abc@example.com'; const channelHandles = ['abc@example.com'];
const email = 'hello@twentytwenty.com'; const email = 'hello@twentytwenty.com';
const blocklist = ['@twenty.com']; const blocklist = ['@twenty.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(false); expect(result).toBe(false);
}); });
it('should return false if email is undefined', () => { it('should return false if email is undefined', () => {
const channelHandle = 'abc@example.com'; const channelHandles = ['abc@example.com'];
const email = undefined; const email = undefined;
const blocklist = ['@twenty.com']; const blocklist = ['@twenty.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(false); expect(result).toBe(false);
}); });
it('should return true if email ends with blocklisted domain', () => { it('should return true if email ends with blocklisted domain', () => {
const channelHandle = 'abc@example.com'; const channelHandles = ['abc@example.com'];
const email = 'hello@twenty.com'; const email = 'hello@twenty.com';
const blocklist = ['@twenty.com']; const blocklist = ['@twenty.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(true); expect(result).toBe(true);
}); });
it('should return false if email is same as channel handle', () => { it('should return false if email is same as channel handle', () => {
const channelHandle = 'hello@twenty.com'; const channelHandles = ['hello@twenty.com'];
const email = 'hello@twenty.com'; const email = 'hello@twenty.com';
const blocklist = ['@twenty.com']; const blocklist = ['@twenty.com'];
const result = isEmailBlocklisted(channelHandle, email, blocklist); const result = isEmailBlocklisted(channelHandles, email, blocklist);
expect(result).toBe(false); expect(result).toBe(false);
}); });

View File

@ -1,11 +1,9 @@
// TODO: Move inside blocklist module
export const isEmailBlocklisted = ( export const isEmailBlocklisted = (
channelHandle: string, channelHandle: string[],
email: string | null | undefined, email: string | null | undefined,
blocklist: string[], blocklist: string[],
): boolean => { ): boolean => {
if (!email || email === channelHandle) { if (!email || channelHandle.includes(email)) {
return false; return false;
} }

View File

@ -4,10 +4,10 @@ import { BlocklistItemDeleteCalendarEventsJob } from 'src/modules/calendar/block
import { BlocklistReimportCalendarEventsJob } from 'src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job'; import { BlocklistReimportCalendarEventsJob } from 'src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job';
import { CalendarBlocklistListener } from 'src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener'; import { CalendarBlocklistListener } from 'src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener';
import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module'; import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module';
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module'; import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
@Module({ @Module({
imports: [CalendarEventCleanerModule, CalendarEventImportManagerModule], imports: [CalendarEventCleanerModule, CalendarCommonModule],
providers: [ providers: [
CalendarBlocklistListener, CalendarBlocklistListener,
BlocklistItemDeleteCalendarEventsJob, BlocklistItemDeleteCalendarEventsJob,

View File

@ -1,20 +1,21 @@
import { Logger, Scope } from '@nestjs/common'; import { Logger, Scope } from '@nestjs/common';
import { Any, ILike } from 'typeorm'; import { And, Any, ILike, In, Not, Or } from 'typeorm';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository'; import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity'; import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service'; import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export type BlocklistItemDeleteCalendarEventsJobData = { export type BlocklistItemDeleteCalendarEventsJobData = WorkspaceEventBatch<
workspaceId: string; ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
blocklistItemId: string; >;
};
@Processor({ @Processor({
queueName: MessageQueue.calendarQueue, queueName: MessageQueue.calendarQueue,
@ -27,77 +28,133 @@ export class BlocklistItemDeleteCalendarEventsJob {
constructor( constructor(
private readonly twentyORMManager: TwentyORMManager, private readonly twentyORMManager: TwentyORMManager,
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly calendarEventCleanerService: CalendarEventCleanerService, private readonly calendarEventCleanerService: CalendarEventCleanerService,
) {} ) {}
@Process(BlocklistItemDeleteCalendarEventsJob.name) @Process(BlocklistItemDeleteCalendarEventsJob.name)
async handle(data: BlocklistItemDeleteCalendarEventsJobData): Promise<void> { async handle(data: BlocklistItemDeleteCalendarEventsJobData): Promise<void> {
const { workspaceId, blocklistItemId } = data; const workspaceId = data.workspaceId;
const blocklistItem = await this.blocklistRepository.getById( const blocklistItemIds = data.events.map(
blocklistItemId, (eventPayload) => eventPayload.recordId,
workspaceId,
); );
if (!blocklistItem) { const blocklistRepository =
this.logger.log( await this.twentyORMManager.getRepository<BlocklistWorkspaceEntity>(
`Blocklist item with id ${blocklistItemId} not found in workspace ${workspaceId}`, 'blocklist',
); );
return; const blocklist = await blocklistRepository.find({
}
const { handle, workspaceMemberId } = blocklistItem;
this.logger.log(
`Deleting calendar events from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
if (!workspaceMemberId) {
throw new Error(
`Workspace member ID is undefined for blocklist item ${blocklistItemId} in workspace ${workspaceId}`,
);
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository('calendarChannel');
const calendarChannels = await calendarChannelRepository.find({
where: { where: {
connectedAccount: { id: Any(blocklistItemIds),
accountOwnerId: workspaceMemberId,
},
}, },
}); });
const calendarChannelIds = calendarChannels.map(({ id }) => id); const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
const isHandleDomain = handle.startsWith('@'); if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
const calendarChannelEventAssociationRepository = const calendarChannelEventAssociationRepository =
await this.twentyORMManager.getRepository( await this.twentyORMManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
'calendarChannelEventAssociation', 'calendarChannelEventAssociation',
); );
await calendarChannelEventAssociationRepository.delete({ for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
calendarEvent: { const handles =
calendarEventParticipants: { handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
handle: isHandleDomain ? ILike(`%${handle}`) : handle,
if (!handles) {
continue;
}
this.logger.log(
`Deleting calendar events from ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const calendarChannels = await calendarChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
}, },
calendarChannelEventAssociations: { where: {
calendarChannelId: Any(calendarChannelIds), connectedAccount: {
accountOwnerId: workspaceMemberId,
},
}, },
}, relations: ['connectedAccount'],
}); });
for (const calendarChannel of calendarChannels) {
const calendarChannelHandles = [calendarChannel.handle];
if (calendarChannel.connectedAccount.handleAliases) {
calendarChannelHandles.push(
...calendarChannel.connectedAccount.handleAliases.split(','),
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(calendarChannelHandles)),
),
}
: { handle };
});
const calendarEventsAssociationsToDelete =
await calendarChannelEventAssociationRepository.find({
where: {
calendarChannelId: calendarChannel.id,
calendarEvent: {
calendarEventParticipants: handleConditions,
},
},
});
if (calendarEventsAssociationsToDelete.length === 0) {
continue;
}
await calendarChannelEventAssociationRepository.delete(
calendarEventsAssociationsToDelete.map(({ id }) => id),
);
}
this.logger.log(
`Deleted calendar events from handle ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
}
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents( await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId, workspaceId,
); );
this.logger.log(
`Deleted calendar events from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
} }
} }

View File

@ -1,23 +1,23 @@
import { Scope } from '@nestjs/common'; import { Scope } from '@nestjs/common';
import { Any } from 'typeorm'; import { Not } from 'typeorm';
import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { import {
CalendarChannelSyncStage, CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity, CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
export type BlocklistReimportCalendarEventsJobData = { export type BlocklistReimportCalendarEventsJobData = WorkspaceEventBatch<
workspaceId: string; ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
workspaceMemberId: string; >;
};
@Processor({ @Processor({
queueName: MessageQueue.calendarQueue, queueName: MessageQueue.calendarQueue,
@ -26,39 +26,38 @@ export type BlocklistReimportCalendarEventsJobData = {
export class BlocklistReimportCalendarEventsJob { export class BlocklistReimportCalendarEventsJob {
constructor( constructor(
private readonly twentyORMManager: TwentyORMManager, private readonly twentyORMManager: TwentyORMManager,
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity) private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly connectedAccountRepository: ConnectedAccountRepository,
) {} ) {}
@Process(BlocklistReimportCalendarEventsJob.name) @Process(BlocklistReimportCalendarEventsJob.name)
async handle(data: BlocklistReimportCalendarEventsJobData): Promise<void> { async handle(data: BlocklistReimportCalendarEventsJobData): Promise<void> {
const { workspaceId, workspaceMemberId } = data; const workspaceId = data.workspaceId;
const connectedAccounts =
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
);
if (!connectedAccounts || connectedAccounts.length === 0) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update( for (const eventPayload of data.events) {
{ const workspaceMemberId =
connectedAccountId: Any( eventPayload.properties.before.workspaceMemberId;
connectedAccounts.map((connectedAccount) => connectedAccount.id),
), const calendarChannels = await calendarChannelRepository.find({
}, select: ['id'],
{ where: {
syncStage: connectedAccount: {
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING, accountOwnerId: workspaceMemberId,
}, },
); syncStage: Not(
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
),
},
});
await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}
} }
} }

View File

@ -31,16 +31,9 @@ export class CalendarBlocklistListener {
ObjectRecordCreateEvent<BlocklistWorkspaceEntity> ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
>, >,
) { ) {
await Promise.all( await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
payload.events.map((eventPayload) => BlocklistItemDeleteCalendarEventsJob.name,
this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>( payload,
BlocklistItemDeleteCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
blocklistItemId: eventPayload.recordId,
},
),
),
); );
} }
@ -50,17 +43,9 @@ export class CalendarBlocklistListener {
ObjectRecordDeleteEvent<BlocklistWorkspaceEntity> ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
>, >,
) { ) {
await Promise.all( await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
payload.events.map((eventPayload) => BlocklistReimportCalendarEventsJob.name,
this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>( payload,
BlocklistReimportCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
workspaceMemberId:
eventPayload.properties.before.workspaceMember.id,
},
),
),
); );
} }
@ -70,31 +55,14 @@ export class CalendarBlocklistListener {
ObjectRecordUpdateEvent<BlocklistWorkspaceEntity> ObjectRecordUpdateEvent<BlocklistWorkspaceEntity>
>, >,
) { ) {
await Promise.all( await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
payload.events.reduce((acc: Promise<void>[], eventPayload) => { BlocklistItemDeleteCalendarEventsJob.name,
acc.push( payload,
this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>( );
BlocklistItemDeleteCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
blocklistItemId: eventPayload.recordId,
},
),
);
acc.push( await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>( BlocklistReimportCalendarEventsJob.name,
BlocklistReimportCalendarEventsJob.name, payload,
{
workspaceId: payload.workspaceId,
workspaceMemberId:
eventPayload.properties.after.workspaceMember.id,
},
),
);
return acc;
}, []),
); );
} }
} }

View File

@ -16,12 +16,13 @@ import { CalendarOngoingStaleCronJob } from 'src/modules/calendar/calendar-event
import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module'; import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module';
import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job'; import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarOngoingStaleJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job'; import { CalendarOngoingStaleJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service'; import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service'; import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service'; import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module'; import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module'; import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
import { RefreshAccessTokenManagerModule } from 'src/modules/connected-account/refresh-access-token-manager/refresh-access-token-manager.module'; import { RefreshAccessTokenManagerModule } from 'src/modules/connected-account/refresh-access-token-manager/refresh-access-token-manager.module';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@ -44,6 +45,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
RefreshAccessTokenManagerModule, RefreshAccessTokenManagerModule,
CalendarEventParticipantManagerModule, CalendarEventParticipantManagerModule,
ConnectedAccountModule, ConnectedAccountModule,
CalendarCommonModule,
], ],
providers: [ providers: [
CalendarChannelSyncStatusService, CalendarChannelSyncStatusService,

View File

@ -6,8 +6,8 @@ import { Process } from 'src/engine/integrations/message-queue/decorators/proces
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { isSyncStale } from 'src/modules/calendar/calendar-event-import-manager/utils/is-sync-stale.util'; import { isSyncStale } from 'src/modules/calendar/calendar-event-import-manager/utils/is-sync-stale.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { import {
CalendarChannelSyncStage, CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity, CalendarChannelWorkspaceEntity,
@ -54,19 +54,19 @@ export class CalendarOngoingStaleJob {
this.logger.log( this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to pending`, `Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to pending`,
); );
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt( await this.calendarChannelSyncStatusService.resetSyncStageStartedAt([
calendarChannel.id, calendarChannel.id,
); ]);
switch (calendarChannel.syncStage) { switch (calendarChannel.syncStage) {
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING: case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING:
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch( await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id, [calendarChannel.id],
); );
break; break;
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING: case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING:
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport( await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
calendarChannel.id, [calendarChannel.id],
); );
break; break;
default: default:

View File

@ -10,7 +10,7 @@ import {
CalendarEventImportException, CalendarEventImportException,
CalendarEventImportExceptionCode, CalendarEventImportExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception'; } from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export enum CalendarEventImportSyncStep { export enum CalendarEventImportSyncStep {
@ -81,7 +81,7 @@ export class CalendarEventImportErrorHandlerService {
calendarChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS calendarChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS
) { ) {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport( await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id, [calendarChannel.id],
workspaceId, workspaceId,
); );
@ -104,19 +104,19 @@ export class CalendarEventImportErrorHandlerService {
switch (syncStep) { switch (syncStep) {
case CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH: case CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.scheduleFullCalendarEventListFetch( await this.calendarChannelSyncStatusService.scheduleFullCalendarEventListFetch(
calendarChannel.id, [calendarChannel.id],
); );
break; break;
case CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH: case CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch( await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id, [calendarChannel.id],
); );
break; break;
case CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT: case CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT:
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport( await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
calendarChannel.id, [calendarChannel.id],
); );
break; break;
@ -130,7 +130,7 @@ export class CalendarEventImportErrorHandlerService {
workspaceId: string, workspaceId: string,
): Promise<void> { ): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport( await this.calendarChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
calendarChannel.id, [calendarChannel.id],
workspaceId, workspaceId,
); );
} }
@ -141,7 +141,7 @@ export class CalendarEventImportErrorHandlerService {
workspaceId: string, workspaceId: string,
): Promise<void> { ): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport( await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id, [calendarChannel.id],
workspaceId, workspaceId,
); );
@ -163,7 +163,7 @@ export class CalendarEventImportErrorHandlerService {
} }
await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch( await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch(
calendarChannel.id, [calendarChannel.id],
workspaceId, workspaceId,
); );
} }

View File

@ -7,7 +7,6 @@ import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository'; import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity'; import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service'; import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { import {
CalendarEventImportErrorHandlerService, CalendarEventImportErrorHandlerService,
CalendarEventImportSyncStep, CalendarEventImportSyncStep,
@ -18,6 +17,7 @@ import {
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service'; import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util'; import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity'; import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { import {
CalendarChannelSyncStage, CalendarChannelSyncStage,
@ -50,7 +50,7 @@ export class CalendarEventsImportService {
: CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH; : CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH;
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing( await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing(
calendarChannel.id, [calendarChannel.id],
); );
let calendarEvents: GetCalendarEventsResponse['calendarEvents'] = []; let calendarEvents: GetCalendarEventsResponse['calendarEvents'] = [];
let nextSyncCursor: GetCalendarEventsResponse['nextSyncCursor'] = ''; let nextSyncCursor: GetCalendarEventsResponse['nextSyncCursor'] = '';
@ -81,7 +81,7 @@ export class CalendarEventsImportService {
); );
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch( await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id, [calendarChannel.id],
); );
} }
@ -92,7 +92,10 @@ export class CalendarEventsImportService {
const { filteredEvents, cancelledEvents } = const { filteredEvents, cancelledEvents } =
filterEventsAndReturnCancelledEvents( filterEventsAndReturnCancelledEvents(
calendarChannel, [
calendarChannel.handle,
...connectedAccount.handleAliases.split(','),
],
calendarEvents, calendarEvents,
blocklist.map((blocklist) => blocklist.handle), blocklist.map((blocklist) => blocklist.handle),
); );
@ -133,8 +136,8 @@ export class CalendarEventsImportService {
}, },
); );
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch( await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialCalendarEventListFetch(
calendarChannel.id, [calendarChannel.id],
); );
} catch (error) { } catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException( await this.calendarEventImportErrorHandlerService.handleDriverException(

View File

@ -1,9 +1,8 @@
import { filterOutBlocklistedEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-out-blocklisted-events.util'; import { filterOutBlocklistedEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-out-blocklisted-events.util';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event'; import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
export const filterEventsAndReturnCancelledEvents = ( export const filterEventsAndReturnCancelledEvents = (
calendarChannel: Pick<CalendarChannelWorkspaceEntity, 'handle'>, calendarChannelHandles: string[],
events: CalendarEventWithParticipants[], events: CalendarEventWithParticipants[],
blocklist: string[], blocklist: string[],
): { ): {
@ -11,7 +10,7 @@ export const filterEventsAndReturnCancelledEvents = (
cancelledEvents: CalendarEventWithParticipants[]; cancelledEvents: CalendarEventWithParticipants[];
} => { } => {
const filteredEvents = filterOutBlocklistedEvents( const filteredEvents = filterOutBlocklistedEvents(
calendarChannel.handle, calendarChannelHandles,
events, events,
blocklist, blocklist,
); );

View File

@ -2,7 +2,7 @@ import { isEmailBlocklisted } from 'src/modules/blocklist/utils/is-email-blockli
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event'; import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
export const filterOutBlocklistedEvents = ( export const filterOutBlocklistedEvents = (
calendarChannelHandle: string, calendarChannelHandles: string[],
events: CalendarEventWithParticipants[], events: CalendarEventWithParticipants[],
blocklist: string[], blocklist: string[],
) => { ) => {
@ -13,7 +13,7 @@ export const filterOutBlocklistedEvents = (
return event.participants.every( return event.participants.every(
(attendee) => (attendee) =>
!isEmailBlocklisted(calendarChannelHandle, attendee.handle, blocklist), !isEmailBlocklisted(calendarChannelHandles, attendee.handle, blocklist),
); );
}); });
}; };

View File

@ -4,6 +4,7 @@ import { CalendarBlocklistManagerModule } from 'src/modules/calendar/blocklist-m
import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module'; import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module';
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module'; import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module'; import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
@Module({ @Module({
imports: [ imports: [
@ -11,6 +12,7 @@ import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/cale
CalendarEventCleanerModule, CalendarEventCleanerModule,
CalendarEventImportManagerModule, CalendarEventImportManagerModule,
CalendarEventParticipantManagerModule, CalendarEventParticipantManagerModule,
CalendarCommonModule,
], ],
providers: [], providers: [],
exports: [], exports: [],

View File

@ -0,0 +1,18 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
@Module({
imports: [
WorkspaceDataSourceModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
ConnectedAccountModule,
],
providers: [CalendarChannelSyncStatusService],
exports: [CalendarChannelSyncStatusService],
})
export class CalendarCommonModule {}

View File

@ -1,13 +1,11 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator'; import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {
CalendarEventImportException,
CalendarEventImportExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
import { import {
CalendarChannelSyncStage, CalendarChannelSyncStage,
CalendarChannelSyncStatus, CalendarChannelSyncStatus,
@ -26,39 +24,55 @@ export class CalendarChannelSyncStatusService {
private readonly accountsToReconnectService: AccountsToReconnectService, private readonly accountsToReconnectService: AccountsToReconnectService,
) {} ) {}
public async scheduleFullCalendarEventListFetch(calendarChannelId: string) { public async scheduleFullCalendarEventListFetch(
calendarChannelIds: string[],
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncStage: syncStage:
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING, CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
}); });
} }
public async schedulePartialCalendarEventListFetch( public async schedulePartialCalendarEventListFetch(
calendarChannelId: string, calendarChannelIds: string[],
) { ) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncStage: syncStage:
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING, CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
}); });
} }
public async markAsCalendarEventListFetchOngoing(calendarChannelId: string) { public async markAsCalendarEventListFetchOngoing(
calendarChannelIds: string[],
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING, syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING, syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(), syncStageStartedAt: new Date().toISOString(),
@ -66,58 +80,92 @@ export class CalendarChannelSyncStatusService {
} }
public async resetAndScheduleFullCalendarEventListFetch( public async resetAndScheduleFullCalendarEventListFetch(
calendarChannelId: string, calendarChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
await this.cacheStorage.del( if (!calendarChannelIds.length) {
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`, return;
); }
for (const calendarChannelId of calendarChannelIds) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncCursor: '', syncCursor: '',
syncStageStartedAt: null, syncStageStartedAt: null,
throttleFailureCount: 0, throttleFailureCount: 0,
}); });
await this.scheduleFullCalendarEventListFetch(calendarChannelId); await this.scheduleFullCalendarEventListFetch(calendarChannelIds);
} }
public async resetSyncStageStartedAt(calendarChannelId: string) { public async resetSyncStageStartedAt(calendarChannelIds: string[]) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncStageStartedAt: null, syncStageStartedAt: null,
}); });
} }
public async scheduleCalendarEventsImport(calendarChannelId: string) { public async scheduleCalendarEventsImport(calendarChannelIds: string[]) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING, syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
}); });
} }
public async markAsCompletedAndSchedulePartialMessageListFetch( public async markAsCalendarEventsImportOngoing(calendarChannelIds: string[]) {
calendarChannelId: string, if (!calendarChannelIds.length) {
) { return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
});
}
public async markAsCompletedAndSchedulePartialCalendarEventListFetch(
calendarChannelIds: string[],
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: syncStage:
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING, CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE, syncStatus: CalendarChannelSyncStatus.ACTIVE,
@ -125,42 +173,53 @@ export class CalendarChannelSyncStatusService {
syncStageStartedAt: null, syncStageStartedAt: null,
}); });
await this.schedulePartialCalendarEventListFetch(calendarChannelId); await this.schedulePartialCalendarEventListFetch(calendarChannelIds);
} }
public async markAsFailedUnknownAndFlushCalendarEventsToImport( public async markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannelId: string, calendarChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await this.cacheStorage.del( for (const calendarChannelId of calendarChannelIds) {
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`, await this.cacheStorage.del(
); `calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
}
await calendarChannelRepository.update(calendarChannelId, { await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN, syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED, syncStage: CalendarChannelSyncStage.FAILED,
}); });
} }
public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport( public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
calendarChannelId: string, calendarChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
await this.cacheStorage.del( for (const calendarChannelId of calendarChannelIds) {
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`, await this.cacheStorage.del(
); `calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
await calendarChannelRepository.update(calendarChannelId, { }
await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS, syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: CalendarChannelSyncStage.FAILED, syncStage: CalendarChannelSyncStage.FAILED,
}); });
@ -170,41 +229,44 @@ export class CalendarChannelSyncStatusService {
'connectedAccount', 'connectedAccount',
); );
const calendarChannel = await calendarChannelRepository.findOne({ const calendarChannels = await calendarChannelRepository.find({
where: { id: calendarChannelId }, select: ['id', 'connectedAccountId'],
where: { id: Any(calendarChannelIds) },
}); });
if (!calendarChannel) { const connectedAccountIds = calendarChannels.map(
throw new CalendarEventImportException( (calendarChannel) => calendarChannel.connectedAccountId,
`Calendar channel ${calendarChannelId} not found in workspace ${workspaceId}`, );
CalendarEventImportExceptionCode.CALENDAR_CHANNEL_NOT_FOUND,
);
}
const connectedAccountId = calendarChannel.connectedAccountId;
await connectedAccountRepository.update( await connectedAccountRepository.update(
{ id: connectedAccountId }, { id: Any(connectedAccountIds) },
{ {
authFailedAt: new Date(), authFailedAt: new Date(),
}, },
); );
await this.addToAccountsToReconnect(calendarChannelId, workspaceId); await this.addToAccountsToReconnect(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
} }
private async addToAccountsToReconnect( private async addToAccountsToReconnect(
calendarChannelId: string, calendarChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository = const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel', 'calendarChannel',
); );
const calendarChannel = await calendarChannelRepository.findOne({ const calendarChannels = await calendarChannelRepository.find({
where: { where: {
id: calendarChannelId, id: Any(calendarChannelIds),
}, },
relations: { relations: {
connectedAccount: { connectedAccount: {
@ -213,18 +275,16 @@ export class CalendarChannelSyncStatusService {
}, },
}); });
if (!calendarChannel) { for (const calendarChannel of calendarChannels) {
return; const userId = calendarChannel.connectedAccount.accountOwner.userId;
const connectedAccountId = calendarChannel.connectedAccount.id;
await this.accountsToReconnectService.addAccountToReconnectByKey(
AccountsToReconnectKeys.ACCOUNTS_TO_RECONNECT_INSUFFICIENT_PERMISSIONS,
userId,
workspaceId,
connectedAccountId,
);
} }
const userId = calendarChannel.connectedAccount.accountOwner.userId;
const connectedAccountId = calendarChannel.connectedAccount.id;
await this.accountsToReconnectService.addAccountToReconnectByKey(
AccountsToReconnectKeys.ACCOUNTS_TO_RECONNECT_INSUFFICIENT_PERMISSIONS,
userId,
workspaceId,
connectedAccountId,
);
} }
} }

View File

@ -1,21 +1,21 @@
import { Logger, Scope } from '@nestjs/common'; import { Logger, Scope } from '@nestjs/common';
import { Any } from 'typeorm'; import { And, Any, ILike, In, Not, Or } from 'typeorm';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository'; import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity'; import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity'; import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
export type BlocklistItemDeleteMessagesJobData = { export type BlocklistItemDeleteMessagesJobData = WorkspaceEventBatch<
workspaceId: string; ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
blocklistItemId: string; >;
};
@Processor({ @Processor({
queueName: MessageQueue.messagingQueue, queueName: MessageQueue.messagingQueue,
@ -25,66 +25,135 @@ export class BlocklistItemDeleteMessagesJob {
private readonly logger = new Logger(BlocklistItemDeleteMessagesJob.name); private readonly logger = new Logger(BlocklistItemDeleteMessagesJob.name);
constructor( constructor(
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly threadCleanerService: MessagingMessageCleanerService, private readonly threadCleanerService: MessagingMessageCleanerService,
private readonly twentyORMManager: TwentyORMManager, private readonly twentyORMManager: TwentyORMManager,
) {} ) {}
@Process(BlocklistItemDeleteMessagesJob.name) @Process(BlocklistItemDeleteMessagesJob.name)
async handle(data: BlocklistItemDeleteMessagesJobData): Promise<void> { async handle(data: BlocklistItemDeleteMessagesJobData): Promise<void> {
const { workspaceId, blocklistItemId } = data; const workspaceId = data.workspaceId;
const blocklistItem = await this.blocklistRepository.getById( const blocklistItemIds = data.events.map(
blocklistItemId, (eventPayload) => eventPayload.recordId,
workspaceId,
); );
if (!blocklistItem) { const blocklistRepository =
this.logger.log( await this.twentyORMManager.getRepository<BlocklistWorkspaceEntity>(
`Blocklist item with id ${blocklistItemId} not found in workspace ${workspaceId}`, 'blocklist',
); );
return; const blocklist = await blocklistRepository.find({
} where: {
id: Any(blocklistItemIds),
},
});
const { handle, workspaceMemberId } = blocklistItem; const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
this.logger.log( if (!acc.has(workspaceMemberId)) {
`Deleting messages from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`, acc.set(workspaceMemberId, []);
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
); );
if (!workspaceMemberId) { const messageChannelRepository =
throw new Error( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
`Workspace member ID is not defined for blocklist item ${blocklistItemId} in workspace ${workspaceId}`, 'messageChannel',
); );
}
const messageChannelMessageAssociationRepository = const messageChannelMessageAssociationRepository =
await this.twentyORMManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
'messageChannelMessageAssociation', 'messageChannelMessageAssociation',
); );
const rolesToDelete: ('from' | 'to')[] = ['from', 'to']; for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
await messageChannelMessageAssociationRepository.delete({ if (!handles) {
messageChannel: { continue;
connectedAccount: { }
accountOwnerId: workspaceMemberId,
this.logger.log(
`Deleting messages from ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const rolesToDelete: ('from' | 'to')[] = ['from', 'to'];
const messageChannels = await messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
}, },
}, where: {
message: { connectedAccount: {
messageParticipants: { accountOwnerId: workspaceMemberId,
handle, },
role: Any(rolesToDelete),
}, },
}, relations: ['connectedAccount'],
}); });
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
if (messageChannel.connectedAccount.handleAliases) {
messageChannelHandles.push(
...messageChannel.connectedAccount.handleAliases.split(','),
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
if (messageChannelMessageAssociationsToDelete.length === 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
this.logger.log(
`Deleted messages from handle ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
}
await this.threadCleanerService.cleanWorkspaceThreads(workspaceId); await this.threadCleanerService.cleanWorkspaceThreads(workspaceId);
this.logger.log(
`Deleted messages from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
} }
} }

View File

@ -0,0 +1,63 @@
import { Scope } from '@nestjs/common';
import { Not } from 'typeorm';
import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
MessageChannelSyncStage,
MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
export type BlocklistReimportMessagesJobData = WorkspaceEventBatch<
ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
>;
@Processor({
queueName: MessageQueue.messagingQueue,
scope: Scope.REQUEST,
})
export class BlocklistReimportMessagesJob {
constructor(
private readonly twentyORMManager: TwentyORMManager,
private readonly messagingChannelSyncStatusService: MessageChannelSyncStatusService,
) {}
@Process(BlocklistReimportMessagesJob.name)
async handle(data: BlocklistReimportMessagesJobData): Promise<void> {
const workspaceId = data.workspaceId;
const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel',
);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const messageChannels = await messageChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
syncStage: Not(
MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
),
},
});
await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
}
}

View File

@ -1,4 +1,4 @@
import { Injectable } from '@nestjs/common'; import { Injectable, Scope } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter'; import { OnEvent } from '@nestjs/event-emitter';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event'; import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
@ -7,28 +7,22 @@ import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/t
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator'; import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type'; import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity'; import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { import {
BlocklistItemDeleteMessagesJob, BlocklistItemDeleteMessagesJob,
BlocklistItemDeleteMessagesJobData, BlocklistItemDeleteMessagesJobData,
} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job'; } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; import {
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; BlocklistReimportMessagesJob,
BlocklistReimportMessagesJobData,
} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job';
@Injectable() @Injectable({ scope: Scope.REQUEST })
export class MessagingBlocklistListener { export class MessagingBlocklistListener {
constructor( constructor(
@InjectMessageQueue(MessageQueue.messagingQueue) @InjectMessageQueue(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService, private readonly messageQueueService: MessageQueueService,
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
private readonly connectedAccountRepository: ConnectedAccountRepository,
private readonly messagingChannelSyncStatusService: MessageChannelSyncStatusService,
private readonly twentyORMManager: TwentyORMManager,
) {} ) {}
@OnEvent('blocklist.created') @OnEvent('blocklist.created')
@ -37,17 +31,9 @@ export class MessagingBlocklistListener {
ObjectRecordCreateEvent<BlocklistWorkspaceEntity> ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
>, >,
) { ) {
await Promise.all( await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
payload.events.map((eventPayload) => BlocklistItemDeleteMessagesJob.name,
// TODO: modify to pass an array of blocklist items payload,
this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
BlocklistItemDeleteMessagesJob.name,
{
workspaceId: payload.workspaceId,
blocklistItemId: eventPayload.recordId,
},
),
),
); );
} }
@ -57,38 +43,10 @@ export class MessagingBlocklistListener {
ObjectRecordDeleteEvent<BlocklistWorkspaceEntity> ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
>, >,
) { ) {
const workspaceId = payload.workspaceId; await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
BlocklistReimportMessagesJob.name,
for (const eventPayload of payload.events) { payload,
const workspaceMemberId = );
eventPayload.properties.before.workspaceMember.id;
const connectedAccount =
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
);
if (!connectedAccount || connectedAccount.length === 0) {
return;
}
const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel',
);
const messageChannel = await messageChannelRepository.findOneOrFail({
where: {
connectedAccountId: connectedAccount[0].id,
},
});
await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
messageChannel.id,
workspaceId,
);
}
} }
@OnEvent('blocklist.updated') @OnEvent('blocklist.updated')
@ -97,45 +55,14 @@ export class MessagingBlocklistListener {
ObjectRecordUpdateEvent<BlocklistWorkspaceEntity> ObjectRecordUpdateEvent<BlocklistWorkspaceEntity>
>, >,
) { ) {
const workspaceId = payload.workspaceId; await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
BlocklistItemDeleteMessagesJob.name,
payload,
);
for (const eventPayload of payload.events) { await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
const workspaceMemberId = BlocklistReimportMessagesJob.name,
eventPayload.properties.before.workspaceMember.id; payload,
);
await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
BlocklistItemDeleteMessagesJob.name,
{
workspaceId,
blocklistItemId: eventPayload.recordId,
},
);
const connectedAccount =
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
);
if (!connectedAccount || connectedAccount.length === 0) {
continue;
}
const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel',
);
const messageChannel = await messageChannelRepository.findOneOrFail({
where: {
connectedAccountId: connectedAccount[0].id,
},
});
await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
messageChannel.id,
workspaceId,
);
}
} }
} }

View File

@ -1,6 +1,7 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job'; import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
import { BlocklistReimportMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job';
import { MessagingBlocklistListener } from 'src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener'; import { MessagingBlocklistListener } from 'src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener';
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module'; import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module';
@ -9,10 +10,8 @@ import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cle
imports: [MessagingCommonModule, MessagingMessageCleanerModule], imports: [MessagingCommonModule, MessagingMessageCleanerModule],
providers: [ providers: [
MessagingBlocklistListener, MessagingBlocklistListener,
{ BlocklistItemDeleteMessagesJob,
provide: BlocklistItemDeleteMessagesJob.name, BlocklistReimportMessagesJob,
useClass: BlocklistItemDeleteMessagesJob,
},
], ],
exports: [], exports: [],
}) })

View File

@ -1,5 +1,7 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator'; import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
@ -12,10 +14,6 @@ import {
MessageChannelSyncStatus, MessageChannelSyncStatus,
MessageChannelWorkspaceEntity, MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import {
MessageImportException,
MessageImportExceptionCode,
} from 'src/modules/messaging/message-import-manager/exceptions/message-import.exception';
@Injectable() @Injectable()
export class MessageChannelSyncStatusService { export class MessageChannelSyncStatusService {
@ -26,216 +24,235 @@ export class MessageChannelSyncStatusService {
private readonly accountsToReconnectService: AccountsToReconnectService, private readonly accountsToReconnectService: AccountsToReconnectService,
) {} ) {}
public async scheduleFullMessageListFetch(messageChannelId: string) { public async scheduleFullMessageListFetch(messageChannelIds: string[]) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStage: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
{ });
syncStage: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
},
);
} }
public async schedulePartialMessageListFetch(messageChannelId: string) { public async schedulePartialMessageListFetch(messageChannelIds: string[]) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStage: MessageChannelSyncStage.PARTIAL_MESSAGE_LIST_FETCH_PENDING,
{ });
syncStage: MessageChannelSyncStage.PARTIAL_MESSAGE_LIST_FETCH_PENDING,
},
);
} }
public async scheduleMessagesImport(messageChannelId: string) { public async scheduleMessagesImport(messageChannelIds: string[]) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
{ });
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
},
);
} }
public async resetAndScheduleFullMessageListFetch( public async resetAndScheduleFullMessageListFetch(
messageChannelId: string, messageChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
await this.cacheStorage.del( if (!messageChannelIds.length) {
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`, return;
); }
for (const messageChannelId of messageChannelIds) {
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
);
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncCursor: '',
{ syncStageStartedAt: null,
syncCursor: '', throttleFailureCount: 0,
syncStageStartedAt: null, });
throttleFailureCount: 0,
},
);
await this.scheduleFullMessageListFetch(messageChannelId); await this.scheduleFullMessageListFetch(messageChannelIds);
} }
public async resetSyncStageStartedAt(messageChannelId: string) { public async resetSyncStageStartedAt(messageChannelIds: string[]) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStageStartedAt: null,
{ });
syncStageStartedAt: null,
},
);
} }
public async markAsMessagesListFetchOngoing(messageChannelId: string) { public async markAsMessagesListFetchOngoing(messageChannelIds: string[]) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
{ syncStatus: MessageChannelSyncStatus.ONGOING,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING, });
syncStatus: MessageChannelSyncStatus.ONGOING,
},
);
} }
public async markAsCompletedAndSchedulePartialMessageListFetch( public async markAsCompletedAndSchedulePartialMessageListFetch(
messageChannelId: string, messageChannelIds: string[],
) { ) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStatus: MessageChannelSyncStatus.ACTIVE,
{ });
syncStatus: MessageChannelSyncStatus.ACTIVE,
},
);
await this.schedulePartialMessageListFetch(messageChannelId); await this.schedulePartialMessageListFetch(messageChannelIds);
} }
public async markAsMessagesImportOngoing(messageChannelId: string) { public async markAsMessagesImportOngoing(messageChannelIds: string[]) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
{ });
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
},
);
} }
public async markAsFailedUnknownAndFlushMessagesToImport( public async markAsFailedUnknownAndFlushMessagesToImport(
messageChannelId: string, messageChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
await this.cacheStorage.del( if (!messageChannelIds.length) {
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`, return;
); }
for (const messageChannelId of messageChannelIds) {
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
);
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStage: MessageChannelSyncStage.FAILED,
{ syncStatus: MessageChannelSyncStatus.FAILED_UNKNOWN,
syncStage: MessageChannelSyncStage.FAILED, });
syncStatus: MessageChannelSyncStatus.FAILED_UNKNOWN,
},
);
} }
public async markAsFailedInsufficientPermissionsAndFlushMessagesToImport( public async markAsFailedInsufficientPermissionsAndFlushMessagesToImport(
messageChannelId: string, messageChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
await this.cacheStorage.del( if (!messageChannelIds.length) {
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`, return;
); }
for (const messageChannelId of messageChannelIds) {
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
);
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
await messageChannelRepository.update( await messageChannelRepository.update(messageChannelIds, {
{ id: messageChannelId }, syncStage: MessageChannelSyncStage.FAILED,
{ syncStatus: MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: MessageChannelSyncStage.FAILED, });
syncStatus: MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
},
);
const connectedAccountRepository = const connectedAccountRepository =
await this.twentyORMManager.getRepository<ConnectedAccountWorkspaceEntity>( await this.twentyORMManager.getRepository<ConnectedAccountWorkspaceEntity>(
'connectedAccount', 'connectedAccount',
); );
const messageChannel = await messageChannelRepository.findOne({ const messageChannels = await messageChannelRepository.find({
where: { id: messageChannelId }, select: ['id', 'connectedAccountId'],
where: { id: Any(messageChannelIds) },
}); });
if (!messageChannel) { const connectedAccountIds = messageChannels.map(
throw new MessageImportException( (messageChannel) => messageChannel.connectedAccountId,
`Message channel ${messageChannelId} not found in workspace ${workspaceId}`, );
MessageImportExceptionCode.MESSAGE_CHANNEL_NOT_FOUND,
);
}
const connectedAccountId = messageChannel.connectedAccountId;
await connectedAccountRepository.update( await connectedAccountRepository.update(
{ id: connectedAccountId }, { id: Any(connectedAccountIds) },
{ {
authFailedAt: new Date(), authFailedAt: new Date(),
}, },
); );
await this.addToAccountsToReconnect(messageChannelId, workspaceId); await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
} }
private async addToAccountsToReconnect( private async addToAccountsToReconnect(
messageChannelId: string, messageChannelIds: string[],
workspaceId: string, workspaceId: string,
) { ) {
if (!messageChannelIds.length) {
return;
}
const messageChannelRepository = const messageChannelRepository =
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>( await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
'messageChannel', 'messageChannel',
); );
const messageChannel = await messageChannelRepository.findOne({ const messageChannels = await messageChannelRepository.find({
where: { id: messageChannelId }, where: { id: Any(messageChannelIds) },
relations: { relations: {
connectedAccount: { connectedAccount: {
accountOwner: true, accountOwner: true,
@ -243,18 +260,16 @@ export class MessageChannelSyncStatusService {
}, },
}); });
if (!messageChannel) { for (const messageChannel of messageChannels) {
return; const userId = messageChannel.connectedAccount.accountOwner.userId;
const connectedAccountId = messageChannel.connectedAccount.id;
await this.accountsToReconnectService.addAccountToReconnectByKey(
AccountsToReconnectKeys.ACCOUNTS_TO_RECONNECT_INSUFFICIENT_PERMISSIONS,
userId,
workspaceId,
connectedAccountId,
);
} }
const userId = messageChannel.connectedAccount.accountOwner.userId;
const connectedAccountId = messageChannel.connectedAccount.id;
await this.accountsToReconnectService.addAccountToReconnectByKey(
AccountsToReconnectKeys.ACCOUNTS_TO_RECONNECT_INSUFFICIENT_PERMISSIONS,
userId,
workspaceId,
connectedAccountId,
);
} }
} }

View File

@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm'; import { EntityManager, IsNull } from 'typeorm';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity'; import { MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
@ -22,67 +22,77 @@ export class MessagingMessageCleanerService {
'message', 'message',
); );
await deleteUsingPagination( const workspaceDataSource = await this.twentyORMManager.getDatasource();
workspaceId,
500, await workspaceDataSource.transaction(async (transactionManager) => {
async ( await deleteUsingPagination(
limit: number, workspaceId,
offset: number, 500,
workspaceId: string, async (
transactionManager?: EntityManager, limit: number,
) => { offset: number,
const nonAssociatedMessages = await messageRepository.find( workspaceId: string,
{ transactionManager: EntityManager,
where: { ) => {
messageChannelMessageAssociations: [], const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
}, },
take: limit, transactionManager,
skip: offset, );
relations: ['messageChannelMessageAssociations'],
},
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id); return nonAssociatedMessages.map(({ id }) => id);
}, },
async ( async (
ids: string[], ids: string[],
workspaceId: string, workspaceId: string,
transactionManager?: EntityManager, transactionManager?: EntityManager,
) => { ) => {
await messageRepository.delete(ids, transactionManager); await messageRepository.delete(ids, transactionManager);
}, },
); transactionManager,
);
await deleteUsingPagination( await deleteUsingPagination(
workspaceId, workspaceId,
500, 500,
async ( async (
limit: number, limit: number,
offset: number, offset: number,
workspaceId: string, workspaceId: string,
transactionManager?: EntityManager, transactionManager?: EntityManager,
) => { ) => {
const orphanThreads = await messageThreadRepository.find( const orphanThreads = await messageThreadRepository.find(
{ {
where: { where: {
messages: [], messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
}, },
take: limit, transactionManager,
skip: offset, );
},
transactionManager,
);
return orphanThreads.map(({ id }) => id); return orphanThreads.map(({ id }) => id);
}, },
async ( async (
ids: string[], ids: string[],
workspaceId: string, workspaceId: string,
transactionManager?: EntityManager, transactionManager?: EntityManager,
) => { ) => {
await messageThreadRepository.delete(ids, transactionManager); await messageThreadRepository.delete(ids, transactionManager);
}, },
); transactionManager,
);
});
} }
} }

View File

@ -55,20 +55,20 @@ export class MessagingOngoingStaleJob {
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`, `Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
); );
await this.messageChannelSyncStatusService.resetSyncStageStartedAt( await this.messageChannelSyncStatusService.resetSyncStageStartedAt([
messageChannel.id, messageChannel.id,
); ]);
switch (messageChannel.syncStage) { switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING: case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
await this.messageChannelSyncStatusService.schedulePartialMessageListFetch( await this.messageChannelSyncStatusService.schedulePartialMessageListFetch(
messageChannel.id, [messageChannel.id],
); );
break; break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING: case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
await this.messageChannelSyncStatusService.scheduleMessagesImport( await this.messageChannelSyncStatusService.scheduleMessagesImport([
messageChannel.id, messageChannel.id,
); ]);
break; break;
default: default:
break; break;

View File

@ -79,7 +79,7 @@ export class MessageImportExceptionHandlerService {
): Promise<void> { ): Promise<void> {
if (messageChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS) { if (messageChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS) {
await this.messageChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport( await this.messageChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
messageChannel.id, [messageChannel.id],
workspaceId, workspaceId,
); );
@ -92,9 +92,7 @@ export class MessageImportExceptionHandlerService {
); );
await messageChannelRepository.increment( await messageChannelRepository.increment(
{ { id: messageChannel.id },
id: messageChannel.id,
},
'throttleFailureCount', 'throttleFailureCount',
1, 1,
); );
@ -102,20 +100,20 @@ export class MessageImportExceptionHandlerService {
switch (syncStep) { switch (syncStep) {
case MessageImportSyncStep.FULL_MESSAGE_LIST_FETCH: case MessageImportSyncStep.FULL_MESSAGE_LIST_FETCH:
await this.messageChannelSyncStatusService.scheduleFullMessageListFetch( await this.messageChannelSyncStatusService.scheduleFullMessageListFetch(
messageChannel.id, [messageChannel.id],
); );
break; break;
case MessageImportSyncStep.PARTIAL_MESSAGE_LIST_FETCH: case MessageImportSyncStep.PARTIAL_MESSAGE_LIST_FETCH:
await this.messageChannelSyncStatusService.schedulePartialMessageListFetch( await this.messageChannelSyncStatusService.schedulePartialMessageListFetch(
messageChannel.id, [messageChannel.id],
); );
break; break;
case MessageImportSyncStep.MESSAGES_IMPORT: case MessageImportSyncStep.MESSAGES_IMPORT:
await this.messageChannelSyncStatusService.scheduleMessagesImport( await this.messageChannelSyncStatusService.scheduleMessagesImport([
messageChannel.id, messageChannel.id,
); ]);
break; break;
default: default:
@ -128,7 +126,7 @@ export class MessageImportExceptionHandlerService {
workspaceId: string, workspaceId: string,
): Promise<void> { ): Promise<void> {
await this.messageChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushMessagesToImport( await this.messageChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushMessagesToImport(
messageChannel.id, [messageChannel.id],
workspaceId, workspaceId,
); );
} }
@ -139,7 +137,7 @@ export class MessageImportExceptionHandlerService {
workspaceId: string, workspaceId: string,
): Promise<void> { ): Promise<void> {
await this.messageChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport( await this.messageChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
messageChannel.id, [messageChannel.id],
workspaceId, workspaceId,
); );
@ -159,7 +157,7 @@ export class MessageImportExceptionHandlerService {
} }
await this.messageChannelSyncStatusService.resetAndScheduleFullMessageListFetch( await this.messageChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
messageChannel.id, [messageChannel.id],
workspaceId, workspaceId,
); );
} }

View File

@ -34,7 +34,7 @@ export class MessagingFullMessageListFetchService {
) { ) {
try { try {
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing( await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
messageChannel.id, [messageChannel.id],
); );
const { messageExternalIds, nextSyncCursor } = const { messageExternalIds, nextSyncCursor } =
@ -95,9 +95,9 @@ export class MessagingFullMessageListFetchService {
}, },
); );
await this.messageChannelSyncStatusService.scheduleMessagesImport( await this.messageChannelSyncStatusService.scheduleMessagesImport([
messageChannel.id, messageChannel.id,
); ]);
} catch (error) { } catch (error) {
await this.messageImportErrorHandlerService.handleDriverException( await this.messageImportErrorHandlerService.handleDriverException(
error, error,

View File

@ -1,7 +1,5 @@
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator'; import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
@ -44,7 +42,6 @@ export class MessagingMessagesImportService {
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity) @InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository, private readonly blocklistRepository: BlocklistRepository,
private readonly emailAliasManagerService: EmailAliasManagerService, private readonly emailAliasManagerService: EmailAliasManagerService,
private readonly isFeatureEnabledService: FeatureFlagService,
private readonly twentyORMManager: TwentyORMManager, private readonly twentyORMManager: TwentyORMManager,
private readonly messagingGetMessagesService: MessagingGetMessagesService, private readonly messagingGetMessagesService: MessagingGetMessagesService,
private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService, private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService,
@ -76,9 +73,9 @@ export class MessagingMessagesImportService {
`Messaging import for workspace ${workspaceId} and account ${connectedAccount.id} starting...`, `Messaging import for workspace ${workspaceId} and account ${connectedAccount.id} starting...`,
); );
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing( await this.messageChannelSyncStatusService.markAsMessagesImportOngoing([
messageChannel.id, messageChannel.id,
); ]);
try { try {
connectedAccount.accessToken = connectedAccount.accessToken =
@ -111,17 +108,10 @@ export class MessagingMessagesImportService {
} }
} }
if ( await this.emailAliasManagerService.refreshHandleAliases(
await this.isFeatureEnabledService.isFeatureEnabled( connectedAccount,
FeatureFlagKey.IsMessagingAliasFetchingEnabled, workspaceId,
workspaceId, );
)
) {
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccount,
workspaceId,
);
}
messageIdsToFetch = await this.cacheStorage.setPop( messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:gmail:${messageChannel.id}`, `messages-to-import:${workspaceId}:gmail:${messageChannel.id}`,
@ -130,7 +120,7 @@ export class MessagingMessagesImportService {
if (!messageIdsToFetch?.length) { if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch( await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch(
messageChannel.id, [messageChannel.id],
); );
return await this.trackMessageImportCompleted( return await this.trackMessageImportCompleted(
@ -151,7 +141,7 @@ export class MessagingMessagesImportService {
); );
const messagesToSave = filterEmails( const messagesToSave = filterEmails(
messageChannel.handle, [messageChannel.handle, ...connectedAccount.handleAliases.split(',')],
allMessages, allMessages,
blocklist.map((blocklistItem) => blocklistItem.handle), blocklist.map((blocklistItem) => blocklistItem.handle),
); );
@ -167,12 +157,12 @@ export class MessagingMessagesImportService {
messageIdsToFetch.length < MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE messageIdsToFetch.length < MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE
) { ) {
await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch( await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch(
messageChannel.id, [messageChannel.id],
); );
} else { } else {
await this.messageChannelSyncStatusService.scheduleMessagesImport( await this.messageChannelSyncStatusService.scheduleMessagesImport([
messageChannel.id, messageChannel.id,
); ]);
} }
const messageChannelRepository = const messageChannelRepository =

View File

@ -38,7 +38,7 @@ export class MessagingPartialMessageListFetchService {
): Promise<void> { ): Promise<void> {
try { try {
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing( await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
messageChannel.id, [messageChannel.id],
); );
const messageChannelRepository = const messageChannelRepository =
@ -70,7 +70,7 @@ export class MessagingPartialMessageListFetchService {
); );
await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch( await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch(
messageChannel.id, [messageChannel.id],
); );
return; return;
@ -110,9 +110,9 @@ export class MessagingPartialMessageListFetchService {
); );
} }
await this.messageChannelSyncStatusService.scheduleMessagesImport( await this.messageChannelSyncStatusService.scheduleMessagesImport([
messageChannel.id, messageChannel.id,
); ]);
} catch (error) { } catch (error) {
await this.messageImportErrorHandlerService.handleDriverException( await this.messageImportErrorHandlerService.handleDriverException(
error, error,

View File

@ -3,19 +3,19 @@ import { MessageWithParticipants } from 'src/modules/messaging/message-import-ma
// Todo: refactor this into several utils // Todo: refactor this into several utils
export const filterEmails = ( export const filterEmails = (
messageChannelHandle: string, messageChannelHandles: string[],
messages: MessageWithParticipants[], messages: MessageWithParticipants[],
blocklist: string[], blocklist: string[],
) => { ) => {
return filterOutBlocklistedMessages( return filterOutBlocklistedMessages(
messageChannelHandle, messageChannelHandles,
filterOutIcsAttachments(messages), filterOutIcsAttachments(messages),
blocklist, blocklist,
); );
}; };
const filterOutBlocklistedMessages = ( const filterOutBlocklistedMessages = (
messageChannelHandle: string, messageChannelHandles: string[],
messages: MessageWithParticipants[], messages: MessageWithParticipants[],
blocklist: string[], blocklist: string[],
) => { ) => {
@ -27,7 +27,7 @@ const filterOutBlocklistedMessages = (
return message.participants.every( return message.participants.every(
(participant) => (participant) =>
!isEmailBlocklisted( !isEmailBlocklisted(
messageChannelHandle, messageChannelHandles,
participant.handle, participant.handle,
blocklist, blocklist,
), ),