set up metrics collecting with open telemetry (#11236)

Done :  
- move metrics and health cache services from health module to metrics
module
- refactor metrics counter from specific method to set up from enum keys
- add OpenTelemetry (Otel) instrumentation for metrics
- set up Otel SDK to send metrics to Otel collector

To do later : 
- implement Otel instrumentation for traces + plug Sentry on top
This commit is contained in:
Etienne
2025-03-28 08:45:24 +01:00
committed by GitHub
parent e9e33c4d29
commit 391392dd87
32 changed files with 575 additions and 297 deletions

View File

@ -3,7 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { HealthModule } from 'src/engine/core-modules/health/health.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
@ -50,7 +50,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
RefreshTokensManagerModule,
ConnectedAccountModule,
CalendarCommonModule,
HealthModule,
MetricsModule,
],
providers: [
CalendarChannelSyncStatusService,

View File

@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { HealthModule } from 'src/engine/core-modules/health/health.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
@ -12,7 +12,7 @@ import { ConnectedAccountModule } from 'src/modules/connected-account/connected-
WorkspaceDataSourceModule,
TypeOrmModule.forFeature([FeatureFlag], 'core'),
ConnectedAccountModule,
HealthModule,
MetricsModule,
],
providers: [CalendarChannelSyncStatusService],
exports: [CalendarChannelSyncStatusService],

View File

@ -5,8 +5,8 @@ import { Any } from 'typeorm';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {
CalendarChannelSyncStage,
@ -24,7 +24,7 @@ export class CalendarChannelSyncStatusService {
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly accountsToReconnectService: AccountsToReconnectService,
private readonly healthCacheService: HealthCacheService,
private readonly metricsService: MetricsService,
) {}
public async scheduleFullCalendarEventListFetch(
@ -179,11 +179,10 @@ export class CalendarChannelSyncStatusService {
await this.schedulePartialCalendarEventListFetch(calendarChannelIds);
await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
CalendarChannelSyncStatus.ACTIVE,
calendarChannelIds,
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.CalendarEventSyncJobActive,
eventIds: calendarChannelIds,
});
}
public async markAsFailedUnknownAndFlushCalendarEventsToImport(
@ -210,11 +209,10 @@ export class CalendarChannelSyncStatusService {
syncStage: CalendarChannelSyncStage.FAILED,
});
await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
CalendarChannelSyncStatus.FAILED_UNKNOWN,
calendarChannelIds,
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.CalendarEventSyncJobFailedUnknown,
eventIds: calendarChannelIds,
});
}
public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
@ -266,11 +264,10 @@ export class CalendarChannelSyncStatusService {
workspaceId,
);
await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
calendarChannelIds,
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.CalendarEventSyncJobFailedInsufficientPermissions,
eventIds: calendarChannelIds,
});
}
private async addToAccountsToReconnect(