mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

better logging errors in messages (#12136)

Browse Source
This commit is contained in:
Guillim
2025-05-20 13:37:24 +02:00
committed by GitHub
parent 0553f58c52
commit ad136b5246
19 changed files with 35 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-item-delete-calendar-events.job.ts
View File

@ -83,12 +83,6 @@ export class BlocklistItemDeleteCalendarEventsJob {
continue;
}
this.logger.log(
`Deleting calendar events from ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const calendarChannels = await calendarChannelRepository.find({
select: {
id: true,
@ -145,12 +139,6 @@ export class BlocklistItemDeleteCalendarEventsJob {
calendarEventsAssociationsToDelete.map(({ id }) => id),
);
}
this.logger.log(
`Deleted calendar events from handle ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
}
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(

14
packages/twenty-server/src/modules/calendar/calendar-event-cleaner/jobs/delete-connected-account-associated-calendar-data.job.ts
View File

@ -1,5 +1,3 @@
import { Logger } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@ -12,10 +10,6 @@ export type DeleteConnectedAccountAssociatedCalendarDataJobData = {
@Processor(MessageQueue.calendarQueue)
export class DeleteConnectedAccountAssociatedCalendarDataJob {
private readonly logger = new Logger(
DeleteConnectedAccountAssociatedCalendarDataJob.name,
);
constructor(
private readonly calendarEventCleanerService: CalendarEventCleanerService,
) {}
@ -24,16 +18,8 @@ export class DeleteConnectedAccountAssociatedCalendarDataJob {
async handle(
data: DeleteConnectedAccountAssociatedCalendarDataJobData,
): Promise<void> {
this.logger.log(
`Deleting connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`,
);
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
data.workspaceId,
);
this.logger.log(
`Deleted connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`,
);
}
}

2
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-ongoing-stale.cron.job.ts
View File

@ -1,7 +1,7 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { Repository } from 'typeorm';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';

10
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job.ts
View File

@ -1,4 +1,4 @@
import { Logger, Scope } from '@nestjs/common';
import { Scope } from '@nestjs/common';
import { In } from 'typeorm';
@ -22,16 +22,13 @@ export type CalendarOngoingStaleJobData = {
scope: Scope.REQUEST,
})
export class CalendarOngoingStaleJob {
private readonly logger = new Logger(CalendarOngoingStaleJob.name);
constructor(
private readonly twentyORMManager: TwentyORMManager,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
) {}
@Process(CalendarOngoingStaleJob.name)
async handle(data: CalendarOngoingStaleJobData): Promise<void> {
const { workspaceId } = data;
async handle(): Promise<void> {
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
@ -51,9 +48,6 @@ export class CalendarOngoingStaleJob {
calendarChannel.syncStageStartedAt &&
isSyncStale(calendarChannel.syncStageStartedAt)
) {
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to pending`,
);
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt([
calendarChannel.id,
]);

14
packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/jobs/calendar-create-company-and-contact-after-sync.job.ts
View File

@ -1,15 +1,15 @@
import { Logger, Scope } from '@nestjs/common';
import { Scope } from '@nestjs/common';
import { IsNull } from 'typeorm';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { CreateCompanyAndContactService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
export type CalendarCreateCompanyAndContactAfterSyncJobData = {
workspaceId: string;
@ -21,9 +21,6 @@ export type CalendarCreateCompanyAndContactAfterSyncJobData = {
scope: Scope.REQUEST,
})
export class CalendarCreateCompanyAndContactAfterSyncJob {
private readonly logger = new Logger(
CalendarCreateCompanyAndContactAfterSyncJob.name,
);
constructor(
private readonly twentyORMManager: TwentyORMManager,
private readonly createCompanyAndContactService: CreateCompanyAndContactService,
@ -33,9 +30,6 @@ export class CalendarCreateCompanyAndContactAfterSyncJob {
async handle(
data: CalendarCreateCompanyAndContactAfterSyncJobData,
): Promise<void> {
this.logger.log(
`create contacts and companies after sync for workspace ${data.workspaceId} and calendarChannel ${data.calendarChannelId}`,
);
const { workspaceId, calendarChannelId } = data;
const calendarChannelRepository =
@ -99,9 +93,5 @@ export class CalendarCreateCompanyAndContactAfterSyncJob {
workspaceId,
FieldActorSource.CALENDAR,
);
this.logger.log(
`create contacts and companies after sync for workspace ${data.workspaceId} and calendarChannel ${data.calendarChannelId} done`,
);
}
}

16
packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts
View File

@ -1,4 +1,4 @@
import { Logger, Scope } from '@nestjs/common';
import { Scope } from '@nestjs/common';
import { And, Any, ILike, In, Not, Or } from 'typeorm';
@ -22,8 +22,6 @@ export type BlocklistItemDeleteMessagesJobData = WorkspaceEventBatch<
scope: Scope.REQUEST,
})
export class BlocklistItemDeleteMessagesJob {
private readonly logger = new Logger(BlocklistItemDeleteMessagesJob.name);
constructor(
private readonly threadCleanerService: MessagingMessageCleanerService,
private readonly twentyORMManager: TwentyORMManager,
@ -81,12 +79,6 @@ export class BlocklistItemDeleteMessagesJob {
continue;
}
this.logger.log(
`Deleting messages from ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const rolesToDelete: ('from' | 'to')[] = ['from', 'to'];
const messageChannels = await messageChannelRepository.find({
@ -146,12 +138,6 @@ export class BlocklistItemDeleteMessagesJob {
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
this.logger.log(
`Deleted messages from handle ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
}
await this.threadCleanerService.cleanWorkspaceThreads(workspaceId);

12
packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts
View File

@ -1,9 +1,9 @@
import { Logger, Scope } from '@nestjs/common';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
export type MessagingConnectedAccountDeletionCleanupJobData = {
workspaceId: string;
@ -27,14 +27,6 @@ export class MessagingConnectedAccountDeletionCleanupJob {
async handle(
data: MessagingConnectedAccountDeletionCleanupJobData,
): Promise<void> {
this.logger.log(
`Deleting connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`,
);
await this.messageCleanerService.cleanWorkspaceThreads(data.workspaceId);
this.logger.log(
`Deleted connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`,
);
}
}

24
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-messages.service.ts
View File

@ -1,4 +1,4 @@
import { Injectable, Logger } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { AxiosResponse } from 'axios';
import { gmail_v1 as gmailV1 } from 'googleapis';
@ -12,8 +12,6 @@ import { MessageWithParticipants } from 'src/modules/messaging/message-import-ma
@Injectable()
export class GmailGetMessagesService {
private readonly logger = new Logger(GmailGetMessagesService.name);
constructor(
private readonly fetchByBatchesService: GmailFetchByBatchService,
private readonly gmailHandleErrorService: GmailHandleErrorService,
@ -25,25 +23,13 @@ export class GmailGetMessagesService {
ConnectedAccountWorkspaceEntity,
'accessToken' | 'refreshToken' | 'id' | 'handle' | 'handleAliases'
>,
workspaceId: string,
): Promise<MessageWithParticipants[]> {
let startTime = Date.now();
const { messageIdsByBatch, batchResponses } =
await this.fetchByBatchesService.fetchAllByBatches(
messageIds,
connectedAccount.accessToken,
'batch_gmail_messages',
);
let endTime = Date.now();
this.logger.log(
`Messaging import for workspace ${workspaceId} and account ${connectedAccount.id} fetching ${
messageIds.length
} messages in ${endTime - startTime}ms`,
);
startTime = Date.now();
const messages = batchResponses.flatMap((response, index) => {
return this.formatBatchResponseAsMessage(
@ -53,14 +39,6 @@ export class GmailGetMessagesService {
);
});
endTime = Date.now();
this.logger.log(
`Messaging import for workspace ${workspaceId} and account ${connectedAccount.id} formatting ${
messageIds.length
} messages in ${endTime - startTime}ms`,
);
return messages;
}

13
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts
View File

@ -1,4 +1,4 @@
import { Injectable } from '@nestjs/common';
import { Injectable, Logger } from '@nestjs/common';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { MicrosoftImportDriverException } from 'src/modules/messaging/message-import-manager/drivers/microsoft/exceptions/microsoft-import-driver.exception';
@ -8,6 +8,7 @@ import { isMicrosoftClientTemporaryError } from 'src/modules/messaging/message-i
@Injectable()
export class MicrosoftFetchByBatchService {
private readonly logger = new Logger(MicrosoftFetchByBatchService.name);
constructor(
private readonly microsoftClientProvider: MicrosoftClientProvider,
) {}
@ -56,8 +57,18 @@ export class MicrosoftFetchByBatchService {
typeof error.body === 'string' &&
isMicrosoftClientTemporaryError(error.body)
) {
// TODO: remove this log once we catch better the error codes
this.logger.error(
`Error temporary (${error.code}) fetching messages for account ${connectedAccount.id.slice(0, 8)}`,
);
this.logger.log(error);
throw new MicrosoftImportDriverException(error.body, error.code, 429);
} else {
// TODO: remove this log once we catch better the error codes
this.logger.error(
`Error unknown (${error.code}) fetching messages for account ${connectedAccount.id.slice(0, 8)}`,
);
this.logger.log(error);
throw error;
}
}

1
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts
View File

@ -51,7 +51,6 @@ xdescribe('Microsoft dev tests : get messages service', () => {
const result = await service.getMessages(
mockMessageIds,
mockConnectedAccount,
'workspace-1',
);
expect(result).toHaveLength(1);

11
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts
View File

@ -28,10 +28,7 @@ export class MicrosoftGetMessagesService {
async getMessages(
messageIds: string[],
connectedAccount: ConnectedAccountType,
workspaceId: string,
): Promise<MessageWithParticipants[]> {
const startTime = Date.now();
try {
const { batchResponses } =
await this.microsoftFetchByBatchService.fetchAllByBatches(
@ -44,14 +41,6 @@ export class MicrosoftGetMessagesService {
connectedAccount,
);
const endTime = Date.now();
this.logger.log(
`Messaging import for workspace ${workspaceId} and account ${
connectedAccount.id
} fetched ${messages.length} messages in ${endTime - startTime}ms`,
);
return messages;
} catch (error) {
this.microsoftHandleErrorService.handleMicrosoftMessageFetchError(error);

5
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-temporary-error.utils.ts
View File

@ -1,3 +1,6 @@
export const isMicrosoftClientTemporaryError = (body: string): boolean => {
return body.includes('Unexpected token < in JSON at position');
return (
body.includes('Unexpected token < in JSON at position') ||
body.includes('ApplicationThrottled 429 error')
);
};

12
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts
View File

@ -1,5 +1,3 @@
import { Logger } from '@nestjs/common';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
@ -14,8 +12,6 @@ export type MessagingCleanCacheJobData = {
@Processor(MessageQueue.messagingQueue)
export class MessagingCleanCacheJob {
private readonly logger = new Logger(MessagingCleanCacheJob.name);
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
@ -23,16 +19,8 @@ export class MessagingCleanCacheJob {
@Process(MessagingCleanCacheJob.name)
async handle(data: MessagingCleanCacheJobData): Promise<void> {
this.logger.log(
`Deleting message channel ${data.messageChannelId} associated cache in workspace ${data.workspaceId}`,
);
await this.cacheStorage.del(
`messages-to-import:${data.workspaceId}:${data.messageChannelId}`,
);
this.logger.log(
`Deleted message channel ${data.messageChannelId} associated cache in workspace ${data.workspaceId}`,
);
}
}

12
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts
View File

@ -1,4 +1,4 @@
import { Logger, Scope } from '@nestjs/common';
import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
@ -31,8 +31,6 @@ export type MessagingMessageListFetchJobData = {
scope: Scope.REQUEST,
})
export class MessagingMessageListFetchJob {
private readonly logger = new Logger(MessagingMessageListFetchJob.name);
constructor(
private readonly messagingFullMessageListFetchService: MessagingFullMessageListFetchService,
private readonly messagingPartialMessageListFetchService: MessagingPartialMessageListFetchService,
@ -117,10 +115,6 @@ export class MessagingMessageListFetchJob {
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.PARTIAL_MESSAGE_LIST_FETCH_PENDING:
this.logger.log(
`Fetching partial message list for workspace ${workspaceId} and messageChannelId ${messageChannel.id}`,
);
await this.messagingMonitoringService.track({
eventName: 'partial_message_list_fetch.started',
workspaceId,
@ -144,10 +138,6 @@ export class MessagingMessageListFetchJob {
break;
case MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING:
this.logger.log(
`Fetching full message list for workspace ${workspaceId} and account ${messageChannel.connectedAccount.id}`,
);
await this.messagingMonitoringService.track({
eventName: 'full_message_list_fetch.started',
workspaceId,

3
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-messages.service.ts
View File

@ -31,20 +31,17 @@ export class MessagingGetMessagesService {
| 'handle'
| 'handleAliases'
>,
workspaceId: string,
): Promise<GetMessagesResponse> {
switch (connectedAccount.provider) {
case ConnectedAccountProvider.GOOGLE:
return this.gmailGetMessagesService.getMessages(
messageIds,
connectedAccount,
workspaceId,
);
case ConnectedAccountProvider.MICROSOFT:
return this.microsoftGetMessagesService.getMessages(
messageIds,
connectedAccount,
workspaceId,
);
default:
throw new MessageImportException(

1
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.spec.ts
View File

@ -214,7 +214,6 @@ describe('MessagingMessagesImportService', () => {
expect(messagingGetMessagesService.getMessages).toHaveBeenCalledWith(
['message-id-1', 'message-id-2'],
mockConnectedAccount,
workspaceId,
);
expect(
saveMessagesService.saveMessagesAndEnqueueContactCreation,

13
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
View File

@ -68,10 +68,6 @@ export class MessagingMessagesImportService {
messageChannelId: messageChannel.id,
});
this.logger.log(
`Messaging import for workspace ${workspaceId} and account ${connectedAccount.id} starting...`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing([
messageChannel.id,
]);
@ -103,6 +99,10 @@ export class MessagingMessagesImportService {
message: error.message,
};
default:
this.logger.error(
`Error (${error.code}) refreshing access token for account ${connectedAccount.id}`,
);
this.logger.log(error);
throw error;
}
}
@ -130,7 +130,6 @@ export class MessagingMessagesImportService {
const allMessages = await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccount,
workspaceId,
);
const blocklist = await this.blocklistRepository.getByWorkspaceMemberId(
@ -184,6 +183,10 @@ export class MessagingMessagesImportService {
workspaceId,
);
} catch (error) {
// TODO: remove this log once we catch better the error codes
this.logger.error(
`Error (${error.code}) importing messages for workspace ${workspaceId.slice(0, 8)} and account ${connectedAccount.id.slice(0, 8)}: ${error.message} - ${error.body}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,

21
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts
View File

@ -1,4 +1,4 @@
import { Injectable, Logger } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { In } from 'typeorm';
@ -20,10 +20,6 @@ import {
@Injectable()
export class MessagingPartialMessageListFetchService {
private readonly logger = new Logger(
MessagingPartialMessageListFetchService.name,
);
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
@ -79,9 +75,6 @@ export class MessagingPartialMessageListFetchService {
);
if (isPartialImportFinished) {
this.logger.log(
`Partial message list import done on message channel ${messageChannel.id} in folder ${folderId} for workspace ${workspaceId} and account ${connectedAccount.id}`,
);
continue;
}
@ -90,10 +83,6 @@ export class MessagingPartialMessageListFetchService {
messageExternalIds,
);
this.logger.log(
`Added ${messageExternalIds.length} messages to import for workspace ${workspaceId} and account ${connectedAccount.id}`,
);
const messageChannelMessageAssociationRepository =
await this.twentyORMManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
'messageChannelMessageAssociation',
@ -110,10 +99,6 @@ export class MessagingPartialMessageListFetchService {
);
}
this.logger.log(
`Deleted ${messageExternalIdsToDelete.length} messages for workspace ${workspaceId} and account ${connectedAccount.id}`,
);
await this.messagingCursorService.updateCursor(
messageChannel,
nextSyncCursor,
@ -130,10 +115,6 @@ export class MessagingPartialMessageListFetchService {
);
if (isPartialImportFinishedForAllFolders) {
this.logger.log(
`Partial message list import done on message channel ${messageChannel.id} entirely for workspace ${workspaceId} and account ${connectedAccount.id}`,
);
await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch(
[messageChannel.id],
);

2
packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.job.ts
View File

@ -38,8 +38,6 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
MESSAGING_MESSAGE_CHANNEL_SYNC_STATUS_MONITORING_CRON_PATTERN,
)
async handle(): Promise<void> {
this.logger.log('Starting message channel sync status monitoring...');
await this.messagingMonitoringService.track({
eventName: 'message_channel.monitoring.sync_status.start',
message: 'Starting message channel sync status monitoring',