admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

6657 Refactor and fix blocklist (#6803)

Browse Source 
Closes #6657
- Fix listeners
- Refactor jobs to take array of events
- Fix calendar events and messages deletion

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Raphaël Bosi
2024-08-31 16:38:47 +02:00
committed by GitHub
parent d9650fd5cf
commit cd66ea74a2
37 changed files with 799 additions and 699 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts
View File

@ -16,12 +16,13 @@ import { CalendarOngoingStaleCronJob } from 'src/modules/calendar/calendar-event
import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module';
import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarOngoingStaleJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
import { RefreshAccessTokenManagerModule } from 'src/modules/connected-account/refresh-access-token-manager/refresh-access-token-manager.module';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@ -44,6 +45,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
RefreshAccessTokenManagerModule,
CalendarEventParticipantManagerModule,
ConnectedAccountModule,
CalendarCommonModule,
],
providers: [
CalendarChannelSyncStatusService,

10
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job.ts
View File

@ -6,8 +6,8 @@ import { Process } from 'src/engine/integrations/message-queue/decorators/proces
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { isSyncStale } from 'src/modules/calendar/calendar-event-import-manager/utils/is-sync-stale.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
@ -54,19 +54,19 @@ export class CalendarOngoingStaleJob {
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to pending`,
);
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt(
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt([
calendarChannel.id,
);
]);
switch (calendarChannel.syncStage) {
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING:
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
break;
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING:
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
calendarChannel.id,
[calendarChannel.id],
);
break;
default:

230
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts
View File

@ -1,230 +0,0 @@
import { Injectable } from '@nestjs/common';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {
CalendarEventImportException,
CalendarEventImportExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
import {
CalendarChannelSyncStage,
CalendarChannelSyncStatus,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { AccountsToReconnectKeys } from 'src/modules/connected-account/types/accounts-to-reconnect-key-value.type';
@Injectable()
export class CalendarChannelSyncStatusService {
constructor(
private readonly twentyORMManager: TwentyORMManager,
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly accountsToReconnectService: AccountsToReconnectService,
) {}
public async scheduleFullCalendarEventListFetch(calendarChannelId: string) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
syncStage:
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
});
}
public async schedulePartialCalendarEventListFetch(
calendarChannelId: string,
) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
syncStage:
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
});
}
public async markAsCalendarEventListFetchOngoing(calendarChannelId: string) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
}
public async resetAndScheduleFullCalendarEventListFetch(
calendarChannelId: string,
workspaceId: string,
) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
});
await this.scheduleFullCalendarEventListFetch(calendarChannelId);
}
public async resetSyncStageStartedAt(calendarChannelId: string) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
syncStageStartedAt: null,
});
}
public async scheduleCalendarEventsImport(calendarChannelId: string) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
});
}
public async markAsCompletedAndSchedulePartialMessageListFetch(
calendarChannelId: string,
) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
syncStage:
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
});
await this.schedulePartialCalendarEventListFetch(calendarChannelId);
}
public async markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannelId: string,
workspaceId: string,
) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
await calendarChannelRepository.update(calendarChannelId, {
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED,
});
}
public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
calendarChannelId: string,
workspaceId: string,
) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
await calendarChannelRepository.update(calendarChannelId, {
syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: CalendarChannelSyncStage.FAILED,
});
const connectedAccountRepository =
await this.twentyORMManager.getRepository<ConnectedAccountWorkspaceEntity>(
'connectedAccount',
);
const calendarChannel = await calendarChannelRepository.findOne({
where: { id: calendarChannelId },
});
if (!calendarChannel) {
throw new CalendarEventImportException(
`Calendar channel ${calendarChannelId} not found in workspace ${workspaceId}`,
CalendarEventImportExceptionCode.CALENDAR_CHANNEL_NOT_FOUND,
);
}
const connectedAccountId = calendarChannel.connectedAccountId;
await connectedAccountRepository.update(
{ id: connectedAccountId },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(calendarChannelId, workspaceId);
}
private async addToAccountsToReconnect(
calendarChannelId: string,
workspaceId: string,
) {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
},
relations: {
connectedAccount: {
accountOwner: true,
},
},
});
if (!calendarChannel) {
return;
}
const userId = calendarChannel.connectedAccount.accountOwner.userId;
const connectedAccountId = calendarChannel.connectedAccount.id;
await this.accountsToReconnectService.addAccountToReconnectByKey(
AccountsToReconnectKeys.ACCOUNTS_TO_RECONNECT_INSUFFICIENT_PERMISSIONS,
userId,
workspaceId,
connectedAccountId,
);
}
}

16
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts
View File

@ -10,7 +10,7 @@ import {
CalendarEventImportException,
CalendarEventImportExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export enum CalendarEventImportSyncStep {
@ -81,7 +81,7 @@ export class CalendarEventImportErrorHandlerService {
calendarChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS
) {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
@ -104,19 +104,19 @@ export class CalendarEventImportErrorHandlerService {
switch (syncStep) {
case CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.scheduleFullCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
break;
case CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
break;
case CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT:
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
calendarChannel.id,
[calendarChannel.id],
);
break;
@ -130,7 +130,7 @@ export class CalendarEventImportErrorHandlerService {
workspaceId: string,
): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
}
@ -141,7 +141,7 @@ export class CalendarEventImportErrorHandlerService {
workspaceId: string,
): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
@ -163,7 +163,7 @@ export class CalendarEventImportErrorHandlerService {
}
await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
}

15
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts
View File

@ -7,7 +7,6 @@ import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import {
CalendarEventImportErrorHandlerService,
CalendarEventImportSyncStep,
@ -18,6 +17,7 @@ import {
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import {
CalendarChannelSyncStage,
@ -50,7 +50,7 @@ export class CalendarEventsImportService {
: CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH;
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing(
calendarChannel.id,
[calendarChannel.id],
);
let calendarEvents: GetCalendarEventsResponse['calendarEvents'] = [];
let nextSyncCursor: GetCalendarEventsResponse['nextSyncCursor'] = '';
@ -81,7 +81,7 @@ export class CalendarEventsImportService {
);
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
}
@ -92,7 +92,10 @@ export class CalendarEventsImportService {
const { filteredEvents, cancelledEvents } =
filterEventsAndReturnCancelledEvents(
calendarChannel,
[
calendarChannel.handle,
...connectedAccount.handleAliases.split(','),
],
calendarEvents,
blocklist.map((blocklist) => blocklist.handle),
);
@ -133,8 +136,8 @@ export class CalendarEventsImportService {
},
);
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch(
calendarChannel.id,
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialCalendarEventListFetch(
[calendarChannel.id],
);
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(

5
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/utils/filter-events.util.ts
View File

@ -1,9 +1,8 @@
import { filterOutBlocklistedEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-out-blocklisted-events.util';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
export const filterEventsAndReturnCancelledEvents = (
calendarChannel: Pick<CalendarChannelWorkspaceEntity, 'handle'>,
calendarChannelHandles: string[],
events: CalendarEventWithParticipants[],
blocklist: string[],
): {
@ -11,7 +10,7 @@ export const filterEventsAndReturnCancelledEvents = (
cancelledEvents: CalendarEventWithParticipants[];
} => {
const filteredEvents = filterOutBlocklistedEvents(
calendarChannel.handle,
calendarChannelHandles,
events,
blocklist,
);

4
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/utils/filter-out-blocklisted-events.util.ts
View File

@ -2,7 +2,7 @@ import { isEmailBlocklisted } from 'src/modules/blocklist/utils/is-email-blockli
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
export const filterOutBlocklistedEvents = (
calendarChannelHandle: string,
calendarChannelHandles: string[],
events: CalendarEventWithParticipants[],
blocklist: string[],
) => {
@ -13,7 +13,7 @@ export const filterOutBlocklistedEvents = (
return event.participants.every(
(attendee) =>
!isEmailBlocklisted(calendarChannelHandle, attendee.handle, blocklist),
!isEmailBlocklisted(calendarChannelHandles, attendee.handle, blocklist),
);
});
};