From 391392dd87d22226353f985c30d578a275a61a58 Mon Sep 17 00:00:00 2001 From: Etienne <45695613+etiennejouan@users.noreply.github.com> Date: Fri, 28 Mar 2025 08:45:24 +0100 Subject: [PATCH] set up metrics collecting with open telemetry (#11236) Done : - move metrics and health cache services from health module to metrics module - refactor metrics counter from specific method to set up from enum keys - add OpenTelemetry (Otel) instrumentation for metrics - set up Otel SDK to send metrics to Otel collector To do later : - implement Otel instrumentation for traces + plug Sentry on top --- ...ngsAdminHealthAccountSyncCountersTable.tsx | 4 - packages/twenty-server/.env.example | 1 + packages/twenty-server/package.json | 3 + .../engine/core-modules/auth/auth.module.ts | 4 +- .../core-modules/captcha/captcha.guard.ts | 10 +- .../environment-variables-group-metadata.ts | 26 +-- .../cast-to-meter-driver.decorator.ts | 21 +++ .../enums/environment-variables-group.enum.ts | 1 + .../environment/environment-variables.ts | 18 ++ .../__tests__/metrics.controller.spec.ts | 30 ---- .../health/controllers/metrics.controller.ts | 28 --- .../health/health-cache.service.ts | 139 --------------- .../core-modules/health/health.module.ts | 9 +- .../connected-account.health.spec.ts | 28 +-- .../indicators/connected-account.health.ts | 18 +- .../types/health-counter-cache-keys.type.ts | 5 - ...account-sync-metrics-by-status.constant.ts | 31 ++++ .../metrics/metrics-cache.service.ts | 94 ++++++++++ .../core-modules/metrics/metrics.module.ts | 10 ++ .../core-modules/metrics/metrics.service.ts | 70 ++++++++ .../metrics/types/meter-driver.type.ts | 4 + .../metrics/types/metrics-keys.type.ts | 9 + packages/twenty-server/src/instrument.ts | 50 +++++- .../calendar-event-import-manager.module.ts | 4 +- .../calendar/common/calendar-common.module.ts | 4 +- .../calendar-channel-sync-status.service.ts | 33 ++-- .../common/messaging-common.module.ts | 4 +- .../message-channel-sync-status.service.ts | 33 ++-- .../src/queue-worker/queue-worker.ts | 2 +- .../src/utils/parse-array-env-var.ts | 13 ++ .../content/developers/self-hosting/setup.mdx | 4 +- yarn.lock | 162 +++++++++++++++++- 32 files changed, 575 insertions(+), 297 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/environment/decorators/cast-to-meter-driver.decorator.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/health/types/health-counter-cache-keys.type.ts create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/constants/account-sync-metrics-by-status.constant.ts create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/metrics.module.ts create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/types/meter-driver.type.ts create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts create mode 100644 packages/twenty-server/src/utils/parse-array-env-var.ts diff --git a/packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminHealthAccountSyncCountersTable.tsx b/packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminHealthAccountSyncCountersTable.tsx index d1b47b3b7..a5995bbca 100644 --- a/packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminHealthAccountSyncCountersTable.tsx +++ b/packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminHealthAccountSyncCountersTable.tsx @@ -21,10 +21,6 @@ export const SettingsAdminHealthAccountSyncCountersTable = ({ } const items = [ - { - label: 'Not Synced', - value: details.counters.NOT_SYNCED, - }, { label: 'Active Sync', value: details.counters.ACTIVE, diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index c02820462..ff6d0b8b7 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -40,6 +40,7 @@ FRONTEND_URL=http://localhost:3001 # LOGGER_DRIVER=console # LOGGER_IS_BUFFER_ENABLED=true # EXCEPTION_HANDLER_DRIVER=sentry +# METER_DRIVER=opentelemetry,console # SENTRY_ENVIRONMENT=main # SENTRY_RELEASE=latest # SENTRY_DSN=https://xxx@xxx.ingest.sentry.io/xxx diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index db7097b30..ce94432f4 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -25,6 +25,9 @@ "@nestjs/devtools-integration": "^0.1.6", "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch", "@node-saml/passport-saml": "^5.0.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.200.0", + "@opentelemetry/sdk-metrics": "^2.0.0", "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch", "@revertdotdev/revert-react": "^0.0.21", "@sentry/nestjs": "^8.30.0", diff --git a/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts b/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts index b21a739b7..3941f30b5 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts @@ -30,9 +30,9 @@ import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.e import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { FileUploadModule } from 'src/engine/core-modules/file/file-upload/file-upload.module'; import { GuardRedirectModule } from 'src/engine/core-modules/guard-redirect/guard-redirect.module'; -import { HealthModule } from 'src/engine/core-modules/health/health.module'; import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module'; import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { OnboardingModule } from 'src/engine/core-modules/onboarding/onboarding.module'; import { WorkspaceSSOModule } from 'src/engine/core-modules/sso/sso.module'; import { WorkspaceSSOIdentityProvider } from 'src/engine/core-modules/sso/workspace-sso-identity-provider.entity'; @@ -90,7 +90,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy'; WorkspaceInvitationModule, EmailVerificationModule, GuardRedirectModule, - HealthModule, + MetricsModule, PermissionsModule, UserRoleModule, ], diff --git a/packages/twenty-server/src/engine/core-modules/captcha/captcha.guard.ts b/packages/twenty-server/src/engine/core-modules/captcha/captcha.guard.ts index 283be0d7b..b8ba9c739 100644 --- a/packages/twenty-server/src/engine/core-modules/captcha/captcha.guard.ts +++ b/packages/twenty-server/src/engine/core-modules/captcha/captcha.guard.ts @@ -7,13 +7,14 @@ import { import { GqlExecutionContext } from '@nestjs/graphql'; import { CaptchaService } from 'src/engine/core-modules/captcha/captcha.service'; -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; @Injectable() export class CaptchaGuard implements CanActivate { constructor( private captchaService: CaptchaService, - private healthCacheService: HealthCacheService, + private metricsService: MetricsService, ) {} async canActivate(context: ExecutionContext): Promise { @@ -26,7 +27,10 @@ export class CaptchaGuard implements CanActivate { if (result.success) { return true; } else { - await this.healthCacheService.updateInvalidCaptchaCache(token); + await this.metricsService.incrementCounter({ + key: MetricsKeys.InvalidCaptcha, + eventId: token, + }); throw new BadRequestException( 'Invalid Captcha, please try another device', diff --git a/packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-group-metadata.ts b/packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-group-metadata.ts index 259759101..bc60b2553 100644 --- a/packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-group-metadata.ts +++ b/packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-group-metadata.ts @@ -54,61 +54,67 @@ export const ENVIRONMENT_VARIABLES_GROUP_METADATA: Record< 'By default, exceptions are sent to the logs. This should be enough for most self-hosting use-cases. For our cloud app we use Sentry.', isHiddenOnLoad: true, }, - [EnvironmentVariablesGroup.Other]: { + [EnvironmentVariablesGroup.Metering]: { position: 900, + description: + 'By default, metrics are sent to the console. OpenTelemetry collector can be set up for self-hosting use-cases.', + isHiddenOnLoad: true, + }, + [EnvironmentVariablesGroup.Other]: { + position: 1000, description: "The variables in this section are mostly used for internal purposes (running our Cloud offering), but shouldn't usually be required for a simple self-hosted instance", isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.BillingConfig]: { - position: 1000, + position: 1100, description: 'We use Stripe in our Cloud app to charge customers. Not relevant to Self-hosters.', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.CaptchaConfig]: { - position: 1100, + position: 1200, description: 'This protects critical endpoints like login and signup with a captcha to prevent bot attacks. Likely unnecessary for self-hosting scenarios.', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.CloudflareConfig]: { - position: 1200, + position: 1300, description: '', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.LLM]: { - position: 1300, + position: 1400, description: 'Configure the LLM provider and model to use for the app. This is experimental and not linked to any public feature.', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.ServerlessConfig]: { - position: 1400, + position: 1500, description: 'In our multi-tenant cloud app, we offload untrusted custom code from workflows to a serverless system (Lambda) for enhanced security and scalability. Self-hosters with a single tenant can typically ignore this configuration.', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.SSL]: { - position: 1500, + position: 1600, description: 'Configure this if you want to setup SSL on your server or full end-to-end encryption. If you just want basic HTTPS, a simple setup like Cloudflare in flexible mode might be easier.', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.SupportChatConfig]: { - position: 1600, + position: 1700, description: 'We use this to setup a small support chat on the bottom left. Currently powered by Front.', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.AnalyticsConfig]: { - position: 1700, + position: 1800, description: 'We’re running a test to perform analytics within the app. This will evolve.', isHiddenOnLoad: true, }, [EnvironmentVariablesGroup.TokensDuration]: { - position: 1800, + position: 1900, description: 'These have been set to sensible default so you probably don’t need to change them unless you have a specific use-case.', isHiddenOnLoad: true, diff --git a/packages/twenty-server/src/engine/core-modules/environment/decorators/cast-to-meter-driver.decorator.ts b/packages/twenty-server/src/engine/core-modules/environment/decorators/cast-to-meter-driver.decorator.ts new file mode 100644 index 000000000..e8d2512ee --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/environment/decorators/cast-to-meter-driver.decorator.ts @@ -0,0 +1,21 @@ +import { Transform } from 'class-transformer'; + +import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type'; + +export const CastToMeterDriverArray = () => + Transform(({ value }: { value: string }) => toMeterDriverArray(value)); + +const toMeterDriverArray = (value: string | undefined) => { + if (typeof value === 'string') { + const rawMeterDrivers = value.split(',').map((driver) => driver.trim()); + const isInvalid = rawMeterDrivers.some( + (driver) => !Object.values(MeterDriver).includes(driver as MeterDriver), + ); + + if (!isInvalid) { + return rawMeterDrivers; + } + } + + return undefined; +}; diff --git a/packages/twenty-server/src/engine/core-modules/environment/enums/environment-variables-group.enum.ts b/packages/twenty-server/src/engine/core-modules/environment/enums/environment-variables-group.enum.ts index 3073a50b1..1019031a6 100644 --- a/packages/twenty-server/src/engine/core-modules/environment/enums/environment-variables-group.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/environment/enums/environment-variables-group.enum.ts @@ -6,6 +6,7 @@ export enum EnvironmentVariablesGroup { MicrosoftAuth = 'microsoft-auth', EmailSettings = 'email-settings', Logging = 'logging', + Metering = 'metering', ExceptionHandler = 'exception-handler', Other = 'other', BillingConfig = 'billing-config', diff --git a/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts b/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts index 266c58219..3f80346a9 100644 --- a/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts @@ -25,6 +25,7 @@ import { LLMTracingDriver } from 'src/engine/core-modules/llm-tracing/interfaces import { CaptchaDriverType } from 'src/engine/core-modules/captcha/interfaces'; import { CastToBoolean } from 'src/engine/core-modules/environment/decorators/cast-to-boolean.decorator'; import { CastToLogLevelArray } from 'src/engine/core-modules/environment/decorators/cast-to-log-level-array.decorator'; +import { CastToMeterDriverArray } from 'src/engine/core-modules/environment/decorators/cast-to-meter-driver.decorator'; import { CastToPositiveNumber } from 'src/engine/core-modules/environment/decorators/cast-to-positive-number.decorator'; import { EnvironmentVariablesMetadata } from 'src/engine/core-modules/environment/decorators/environment-variables-metadata.decorator'; import { IsAWSRegion } from 'src/engine/core-modules/environment/decorators/is-aws-region.decorator'; @@ -36,6 +37,7 @@ import { EnvironmentVariablesGroup } from 'src/engine/core-modules/environment/e import { ExceptionHandlerDriver } from 'src/engine/core-modules/exception-handler/interfaces'; import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces'; import { LoggerDriverType } from 'src/engine/core-modules/logger/interfaces'; +import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type'; import { ServerlessDriverType } from 'src/engine/core-modules/serverless/serverless.interface'; export class EnvironmentVariables { @@ -585,6 +587,22 @@ export class EnvironmentVariables { @IsOptional() LOG_LEVELS: LogLevel[] = ['log', 'error', 'warn']; + @EnvironmentVariablesMetadata({ + group: EnvironmentVariablesGroup.Metering, + description: 'Driver used for collect metrics (OpenTelemetry or Console)', + }) + @CastToMeterDriverArray() + @IsOptional() + METER_DRIVER: MeterDriver[] = [MeterDriver.Console]; + + @EnvironmentVariablesMetadata({ + group: EnvironmentVariablesGroup.Metering, + description: 'Endpoint URL for the OpenTelemetry collector', + }) + @ValidateIf((env) => env.METER_DRIVER.includes(MeterDriver.OpenTelemetry)) + @IsOptional() + OTLP_COLLECTOR_ENDPOINT_URL: string; + @EnvironmentVariablesMetadata({ group: EnvironmentVariablesGroup.ExceptionHandler, description: 'Driver used for logging (only console for now)', diff --git a/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts deleted file mode 100644 index 72dd5b86a..000000000 --- a/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller'; -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; - -describe('MetricsController', () => { - let metricsController: MetricsController; - - beforeEach(async () => { - const testingModule: TestingModule = await Test.createTestingModule({ - controllers: [MetricsController], - providers: [ - { - provide: HealthCacheService, - useValue: { - getMessageChannelSyncJobByStatusCounter: jest.fn(), - getCalendarChannelSyncJobByStatusCounter: jest.fn(), - getInvalidCaptchaCounter: jest.fn(), - }, - }, - ], - }).compile(); - - metricsController = testingModule.get(MetricsController); - }); - - it('should be defined', () => { - expect(metricsController).toBeDefined(); - }); -}); diff --git a/packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts deleted file mode 100644 index 50d2cb2d3..000000000 --- a/packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Controller, Get } from '@nestjs/common'; - -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; -import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type'; - -@Controller('metrics') -export class MetricsController { - constructor(private readonly healthCacheService: HealthCacheService) {} - - @Get('/message-channel-sync-job-by-status-counter') - getMessageChannelSyncJobByStatusCounter() { - return this.healthCacheService.countChannelSyncJobByStatus( - HealthCounterCacheKeys.MessageChannelSyncJobByStatus, - ); - } - - @Get('/invalid-captcha-counter') - getInvalidCaptchaCounter() { - return this.healthCacheService.getInvalidCaptchaCounter(); - } - - @Get('/calendar-channel-sync-job-by-status-counter') - getCalendarChannelSyncJobByStatusCounter() { - return this.healthCacheService.countChannelSyncJobByStatus( - HealthCounterCacheKeys.CalendarEventSyncJobByStatus, - ); - } -} diff --git a/packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts b/packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts deleted file mode 100644 index ed8779822..000000000 --- a/packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; -import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; -import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; -import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; -import { AccountSyncJobByStatusCounter } from 'src/engine/core-modules/health/types/account-sync-metrics.types'; -import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type'; -import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; -import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; - -const CACHE_BUCKET_DURATION_MS = 15000; // 15 seconds window for each cache bucket - -@Injectable() -export class HealthCacheService { - private readonly healthMetricsTimeWindowInMinutes: number; - private readonly healthCacheTtl: number; - - constructor( - @InjectCacheStorage(CacheStorageNamespace.EngineHealth) - private readonly cacheStorage: CacheStorageService, - private readonly environmentService: EnvironmentService, - ) { - this.healthMetricsTimeWindowInMinutes = this.environmentService.get( - 'HEALTH_METRICS_TIME_WINDOW_IN_MINUTES', - ); - this.healthCacheTtl = this.healthMetricsTimeWindowInMinutes * 60000 * 2; - } - - private getCacheBucketStartTimestamp(timestamp: number): number { - return ( - Math.floor(timestamp / CACHE_BUCKET_DURATION_MS) * - CACHE_BUCKET_DURATION_MS - ); - } - - private getCacheKeyWithTimestamp(key: string, timestamp?: number): string { - const currentIntervalTimestamp = - timestamp ?? this.getCacheBucketStartTimestamp(Date.now()); - - return `${key}:${currentIntervalTimestamp}`; - } - - private getLastCacheBucketStartTimestampsFromDate( - cacheBucketsCount: number, - date: number = Date.now(), - ): number[] { - const currentIntervalTimestamp = this.getCacheBucketStartTimestamp(date); - - return Array.from( - { length: cacheBucketsCount }, - (_, i) => currentIntervalTimestamp - i * CACHE_BUCKET_DURATION_MS, - ); - } - - async updateMessageOrCalendarChannelSyncJobByStatusCache( - key: HealthCounterCacheKeys, - status: MessageChannelSyncStatus | CalendarChannelSyncStatus, - messageChannelIds: string[], - ) { - return await this.cacheStorage.setAdd( - this.getCacheKeyWithTimestamp(`${key}:${status}`), - messageChannelIds, - this.healthCacheTtl, - ); - } - - async countChannelSyncJobByStatus( - key: - | HealthCounterCacheKeys.MessageChannelSyncJobByStatus - | HealthCounterCacheKeys.CalendarEventSyncJobByStatus, - timeWindowInSeconds: number = this.healthMetricsTimeWindowInMinutes * 60, - ): Promise { - if ((timeWindowInSeconds * 1000) % CACHE_BUCKET_DURATION_MS !== 0) { - throw new Error( - `Time window must be divisible by ${CACHE_BUCKET_DURATION_MS}`, - ); - } - - const now = Date.now(); - const countByStatus = {} as AccountSyncJobByStatusCounter; - const statuses = - key === HealthCounterCacheKeys.MessageChannelSyncJobByStatus - ? Object.values(MessageChannelSyncStatus).filter( - (status) => status !== MessageChannelSyncStatus.ONGOING, - ) - : Object.values(CalendarChannelSyncStatus).filter( - (status) => status !== CalendarChannelSyncStatus.ONGOING, - ); - - const cacheBuckets = - timeWindowInSeconds / (CACHE_BUCKET_DURATION_MS / 1000); - - for (const status of statuses) { - const cacheKeys = this.computeTimeStampedCacheKeys( - `${key}:${status}`, - cacheBuckets, - now, - ); - - const channelIdsCount = - await this.cacheStorage.countAllSetMembers(cacheKeys); - - countByStatus[status] = channelIdsCount; - } - - return countByStatus; - } - - computeTimeStampedCacheKeys( - key: string, - cacheBucketsCount: number, - date: number = Date.now(), - ) { - return this.getLastCacheBucketStartTimestampsFromDate( - cacheBucketsCount, - date, - ).map((timestamp) => this.getCacheKeyWithTimestamp(key, timestamp)); - } - - async updateInvalidCaptchaCache(captchaToken: string) { - return await this.cacheStorage.setAdd( - this.getCacheKeyWithTimestamp(HealthCounterCacheKeys.InvalidCaptcha), - [captchaToken], - this.healthCacheTtl, - ); - } - - async getInvalidCaptchaCounter( - timeWindowInSeconds: number = this.healthMetricsTimeWindowInMinutes * 60, - ) { - return await this.cacheStorage.countAllSetMembers( - this.computeTimeStampedCacheKeys( - HealthCounterCacheKeys.InvalidCaptcha, - timeWindowInSeconds / (CACHE_BUCKET_DURATION_MS / 1000), - ), - ); - } -} diff --git a/packages/twenty-server/src/engine/core-modules/health/health.module.ts b/packages/twenty-server/src/engine/core-modules/health/health.module.ts index fb1566bca..7cb971f80 100644 --- a/packages/twenty-server/src/engine/core-modules/health/health.module.ts +++ b/packages/twenty-server/src/engine/core-modules/health/health.module.ts @@ -3,14 +3,12 @@ import { TerminusModule } from '@nestjs/terminus'; import { TypeOrmModule } from '@nestjs/typeorm'; import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller'; -import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller'; import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; -import { HealthCacheService } from './health-cache.service'; - import { ConnectedAccountHealth } from './indicators/connected-account.health'; import { DatabaseHealthIndicator } from './indicators/database.health'; import { RedisHealthIndicator } from './indicators/redis.health'; @@ -21,10 +19,10 @@ import { WorkerHealthIndicator } from './indicators/worker.health'; RedisClientModule, WorkspaceMigrationModule, TypeOrmModule.forFeature([Workspace], 'core'), + MetricsModule, ], - controllers: [HealthController, MetricsController], + controllers: [HealthController], providers: [ - HealthCacheService, DatabaseHealthIndicator, RedisHealthIndicator, WorkerHealthIndicator, @@ -32,7 +30,6 @@ import { WorkerHealthIndicator } from './indicators/worker.health'; AppHealthIndicator, ], exports: [ - HealthCacheService, DatabaseHealthIndicator, RedisHealthIndicator, WorkerHealthIndicator, diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts index b920c6719..7c578238d 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts @@ -4,19 +4,19 @@ import { Test, TestingModule } from '@nestjs/testing'; import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const'; -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; describe('ConnectedAccountHealth', () => { let service: ConnectedAccountHealth; - let healthCacheService: jest.Mocked; + let metricsService: jest.Mocked; let healthIndicatorService: jest.Mocked; beforeEach(async () => { - healthCacheService = { - countChannelSyncJobByStatus: jest.fn(), + metricsService = { + groupMetrics: jest.fn(), } as any; healthIndicatorService = { @@ -41,8 +41,8 @@ describe('ConnectedAccountHealth', () => { providers: [ ConnectedAccountHealth, { - provide: HealthCacheService, - useValue: healthCacheService, + provide: MetricsService, + useValue: metricsService, }, { provide: HealthIndicatorService, @@ -64,7 +64,7 @@ describe('ConnectedAccountHealth', () => { describe('message sync health', () => { it('should return up status when no message sync jobs are present', async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce({ [MessageChannelSyncStatus.NOT_SYNCED]: 0, [MessageChannelSyncStatus.ACTIVE]: 0, @@ -92,7 +92,7 @@ describe('ConnectedAccountHealth', () => { }); it(`should return down status when message sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce({ [MessageChannelSyncStatus.NOT_SYNCED]: 0, [MessageChannelSyncStatus.ACTIVE]: 1, @@ -122,7 +122,7 @@ describe('ConnectedAccountHealth', () => { describe('calendar sync health', () => { it('should return up status when no calendar sync jobs are present', async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce({ [MessageChannelSyncStatus.NOT_SYNCED]: 0, [MessageChannelSyncStatus.ACTIVE]: 0, @@ -150,7 +150,7 @@ describe('ConnectedAccountHealth', () => { }); it(`should return down status when calendar sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce({ [MessageChannelSyncStatus.NOT_SYNCED]: 0, [MessageChannelSyncStatus.ACTIVE]: 1, @@ -180,7 +180,7 @@ describe('ConnectedAccountHealth', () => { describe('timeout handling', () => { it('should handle message sync timeout', async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce( new Promise((resolve) => setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100), @@ -207,7 +207,7 @@ describe('ConnectedAccountHealth', () => { }); it('should handle calendar sync timeout', async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce({ [MessageChannelSyncStatus.NOT_SYNCED]: 0, [MessageChannelSyncStatus.ACTIVE]: 1, @@ -236,7 +236,7 @@ describe('ConnectedAccountHealth', () => { describe('combined health check', () => { it('should return combined status with both checks healthy', async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce({ [MessageChannelSyncStatus.NOT_SYNCED]: 0, [MessageChannelSyncStatus.ACTIVE]: 8, @@ -256,7 +256,7 @@ describe('ConnectedAccountHealth', () => { }); it('should return down status when both syncs fail', async () => { - healthCacheService.countChannelSyncJobByStatus + metricsService.groupMetrics .mockResolvedValueOnce({ [MessageChannelSyncStatus.NOT_SYNCED]: 0, [MessageChannelSyncStatus.ACTIVE]: 1, diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts index 96c3d6dca..95f1bb53e 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts @@ -6,15 +6,17 @@ import { import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const'; -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; -import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type'; import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; - +import { + CALENDAR_SYNC_METRICS_BY_STATUS, + MESSAGE_SYNC_METRICS_BY_STATUS, +} from 'src/engine/core-modules/metrics/constants/account-sync-metrics-by-status.constant'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; @Injectable() export class ConnectedAccountHealth { constructor( private readonly healthIndicatorService: HealthIndicatorService, - private readonly healthCacheService: HealthCacheService, + private readonly metricsService: MetricsService, ) {} private async checkMessageSyncHealth(): Promise { @@ -22,9 +24,7 @@ export class ConnectedAccountHealth { try { const counters = await withHealthCheckTimeout( - this.healthCacheService.countChannelSyncJobByStatus( - HealthCounterCacheKeys.MessageChannelSyncJobByStatus, - ), + this.metricsService.groupMetrics(MESSAGE_SYNC_METRICS_BY_STATUS), HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT, ); @@ -73,9 +73,7 @@ export class ConnectedAccountHealth { try { const counters = await withHealthCheckTimeout( - this.healthCacheService.countChannelSyncJobByStatus( - HealthCounterCacheKeys.CalendarEventSyncJobByStatus, - ), + this.metricsService.groupMetrics(CALENDAR_SYNC_METRICS_BY_STATUS), HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT, ); diff --git a/packages/twenty-server/src/engine/core-modules/health/types/health-counter-cache-keys.type.ts b/packages/twenty-server/src/engine/core-modules/health/types/health-counter-cache-keys.type.ts deleted file mode 100644 index ecc4fde3c..000000000 --- a/packages/twenty-server/src/engine/core-modules/health/types/health-counter-cache-keys.type.ts +++ /dev/null @@ -1,5 +0,0 @@ -export enum HealthCounterCacheKeys { - MessageChannelSyncJobByStatus = 'message-channel-sync-job-by-status', - InvalidCaptcha = 'invalid-captcha', - CalendarEventSyncJobByStatus = 'calendar-event-sync-job-by-status', -} diff --git a/packages/twenty-server/src/engine/core-modules/metrics/constants/account-sync-metrics-by-status.constant.ts b/packages/twenty-server/src/engine/core-modules/metrics/constants/account-sync-metrics-by-status.constant.ts new file mode 100644 index 000000000..815462526 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/constants/account-sync-metrics-by-status.constant.ts @@ -0,0 +1,31 @@ +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; + +export const MESSAGE_SYNC_METRICS_BY_STATUS = [ + { + name: 'ACTIVE', + cacheKey: MetricsKeys.MessageChannelSyncJobActive, + }, + { + name: 'FAILED_UNKNOWN', + cacheKey: MetricsKeys.MessageChannelSyncJobFailedUnknown, + }, + { + name: 'FAILED_INSUFFICIENT_PERMISSIONS', + cacheKey: MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions, + }, +]; + +export const CALENDAR_SYNC_METRICS_BY_STATUS = [ + { + name: 'ACTIVE', + cacheKey: MetricsKeys.CalendarEventSyncJobActive, + }, + { + name: 'FAILED_UNKNOWN', + cacheKey: MetricsKeys.CalendarEventSyncJobFailedUnknown, + }, + { + name: 'FAILED_INSUFFICIENT_PERMISSIONS', + cacheKey: MetricsKeys.CalendarEventSyncJobFailedInsufficientPermissions, + }, +]; diff --git a/packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts new file mode 100644 index 000000000..9666aef8d --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts @@ -0,0 +1,94 @@ +import { Injectable } from '@nestjs/common'; + +import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; +import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; +import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; +import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; + +const CACHE_BUCKET_DURATION_MS = 15000; // 15 seconds window for each cache bucket + +@Injectable() +export class MetricsCacheService { + private readonly healthMetricsTimeWindowInMinutes: number; + private readonly healthCacheTtl: number; + + constructor( + @InjectCacheStorage(CacheStorageNamespace.EngineHealth) + private readonly cacheStorage: CacheStorageService, + private readonly environmentService: EnvironmentService, + ) { + this.healthMetricsTimeWindowInMinutes = this.environmentService.get( + 'HEALTH_METRICS_TIME_WINDOW_IN_MINUTES', + ); + this.healthCacheTtl = this.healthMetricsTimeWindowInMinutes * 60000 * 2; + } + + private getCacheBucketStartTimestamp(timestamp: number): number { + return ( + Math.floor(timestamp / CACHE_BUCKET_DURATION_MS) * + CACHE_BUCKET_DURATION_MS + ); + } + + private getCacheKeyWithTimestamp(key: string, timestamp?: number): string { + const currentIntervalTimestamp = + timestamp ?? this.getCacheBucketStartTimestamp(Date.now()); + + return `${key}:${currentIntervalTimestamp}`; + } + + private getLastCacheBucketStartTimestampsFromDate( + cacheBucketsCount: number, + date: number, + ): number[] { + const currentIntervalTimestamp = this.getCacheBucketStartTimestamp(date); + + return Array.from( + { length: cacheBucketsCount }, + (_, i) => currentIntervalTimestamp - i * CACHE_BUCKET_DURATION_MS, + ); + } + + async updateCounter(key: MetricsKeys, items: string[]) { + return await this.cacheStorage.setAdd( + this.getCacheKeyWithTimestamp(key), + items, + this.healthCacheTtl, + ); + } + + async computeCount({ + key, + timeWindowInSeconds = this.healthMetricsTimeWindowInMinutes * 60, + date = Date.now(), + }: { + key: MetricsKeys; + timeWindowInSeconds?: number; + date?: number; + }): Promise { + if ((timeWindowInSeconds * 1000) % CACHE_BUCKET_DURATION_MS !== 0) { + throw new Error( + `Time window must be divisible by ${CACHE_BUCKET_DURATION_MS}`, + ); + } + + const cacheBuckets = + timeWindowInSeconds / (CACHE_BUCKET_DURATION_MS / 1000); + + const cacheKeys = this.computeTimeStampedCacheKeys(key, cacheBuckets, date); + + return await this.cacheStorage.countAllSetMembers(cacheKeys); + } + + computeTimeStampedCacheKeys( + key: string, + cacheBucketsCount: number, + date: number, + ) { + return this.getLastCacheBucketStartTimestampsFromDate( + cacheBucketsCount, + date, + ).map((timestamp) => this.getCacheKeyWithTimestamp(key, timestamp)); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/metrics/metrics.module.ts b/packages/twenty-server/src/engine/core-modules/metrics/metrics.module.ts new file mode 100644 index 000000000..a0afac6e1 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/metrics.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; + +import { MetricsCacheService } from 'src/engine/core-modules/metrics/metrics-cache.service'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; + +@Module({ + providers: [MetricsService, MetricsCacheService], + exports: [MetricsService, MetricsCacheService], +}) +export class MetricsModule {} diff --git a/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts new file mode 100644 index 000000000..beeec81eb --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts @@ -0,0 +1,70 @@ +import { Injectable } from '@nestjs/common'; + +import { metrics } from '@opentelemetry/api'; + +import { MetricsCacheService } from 'src/engine/core-modules/metrics/metrics-cache.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; + +@Injectable() +export class MetricsService { + constructor(private readonly metricsCacheService: MetricsCacheService) {} + + async incrementCounter({ + key, + eventId, + shouldStoreInCache = true, + }: { + key: MetricsKeys; + eventId: string; + shouldStoreInCache?: boolean; + }) { + //TODO : Define meter name usage in monitoring + const meter = metrics.getMeter('twenty-server'); + const counter = meter.createCounter(key); + + counter.add(1); + + if (shouldStoreInCache) { + this.metricsCacheService.updateCounter(key, [eventId]); + } + } + + async batchIncrementCounter({ + key, + eventIds, + shouldStoreInCache = true, + }: { + key: MetricsKeys; + eventIds: string[]; + shouldStoreInCache?: boolean; + }) { + //TODO : Define meter name usage in monitoring + const meter = metrics.getMeter('twenty-server'); + const counter = meter.createCounter(key); + + counter.add(eventIds.length); + + if (shouldStoreInCache) { + this.metricsCacheService.updateCounter(key, eventIds); + } + } + + async groupMetrics( + metrics: { name: string; cacheKey: MetricsKeys }[], + ): Promise> { + const groupedMetrics: Record = {}; + + const date = Date.now(); + + for (const metric of metrics) { + const metricValue = await this.metricsCacheService.computeCount({ + key: metric.cacheKey, + date, + }); + + groupedMetrics[metric.name] = metricValue; + } + + return groupedMetrics; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/meter-driver.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/meter-driver.type.ts new file mode 100644 index 000000000..fb8be359b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/types/meter-driver.type.ts @@ -0,0 +1,4 @@ +export enum MeterDriver { + OpenTelemetry = 'opentelemetry', + Console = 'console', +} diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts new file mode 100644 index 000000000..c0420eaac --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts @@ -0,0 +1,9 @@ +export enum MetricsKeys { + MessageChannelSyncJobActive = 'message-channel-sync-job/active', + MessageChannelSyncJobFailedInsufficientPermissions = 'message-channel-sync-job/failed-insufficient-permissions', + MessageChannelSyncJobFailedUnknown = 'message-channel-sync-job/failed-unknown', + CalendarEventSyncJobActive = 'calendar-event-sync-job/active', + CalendarEventSyncJobFailedInsufficientPermissions = 'calendar-event-sync-job/failed-insufficient-permissions', + CalendarEventSyncJobFailedUnknown = 'calendar-event-sync-job/failed-unknown', + InvalidCaptcha = 'invalid-captcha', +} diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.ts index bd179aa56..5a983e788 100644 --- a/packages/twenty-server/src/instrument.ts +++ b/packages/twenty-server/src/instrument.ts @@ -1,10 +1,27 @@ -import * as Sentry from '@sentry/nestjs'; +import process from 'process'; + +import opentelemetry from '@opentelemetry/api'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { + ConsoleMetricExporter, + MeterProvider, + PeriodicExportingMetricReader, +} from '@opentelemetry/sdk-metrics'; +import * as Sentry from '@sentry/node'; import { nodeProfilingIntegration } from '@sentry/profiling-node'; import { NodeEnvironment } from 'src/engine/core-modules/environment/interfaces/node-environment.interface'; import { ExceptionHandlerDriver } from 'src/engine/core-modules/exception-handler/interfaces'; +import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type'; import { WorkspaceCacheKeys } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; +import { parseArrayEnvVar } from 'src/utils/parse-array-env-var'; + +const meterDrivers = parseArrayEnvVar( + process.env.METER_DRIVER, + Object.values(MeterDriver), + [MeterDriver.Console], +); if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry) { Sentry.init({ @@ -29,3 +46,34 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry) { debug: process.env.NODE_ENV === NodeEnvironment.development, }); } + +// Meter setup + +const meterProvider = new MeterProvider({ + readers: [ + ...(meterDrivers.includes(MeterDriver.Console) + ? [ + new PeriodicExportingMetricReader({ + exporter: new ConsoleMetricExporter(), + exportIntervalMillis: 10000, + }), + ] + : []), + ...(meterDrivers.includes(MeterDriver.OpenTelemetry) + ? [ + new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: process.env.OTLP_COLLECTOR_METRICS_ENDPOINT_URL, + }), + exportIntervalMillis: 10000, + }), + ] + : []), + ], +}); + +opentelemetry.metrics.setGlobalMeterProvider(meterProvider); + +process.on('SIGTERM', async () => { + await meterProvider.shutdown(); +}); diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts index 40af5bdaa..fa8813256 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts @@ -3,7 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { BillingModule } from 'src/engine/core-modules/billing/billing.module'; import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; -import { HealthModule } from 'src/engine/core-modules/health/health.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity'; import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module'; @@ -50,7 +50,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta RefreshTokensManagerModule, ConnectedAccountModule, CalendarCommonModule, - HealthModule, + MetricsModule, ], providers: [ CalendarChannelSyncStatusService, diff --git a/packages/twenty-server/src/modules/calendar/common/calendar-common.module.ts b/packages/twenty-server/src/modules/calendar/common/calendar-common.module.ts index 09074f27e..4ca677624 100644 --- a/packages/twenty-server/src/modules/calendar/common/calendar-common.module.ts +++ b/packages/twenty-server/src/modules/calendar/common/calendar-common.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; -import { HealthModule } from 'src/engine/core-modules/health/health.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module'; @@ -12,7 +12,7 @@ import { ConnectedAccountModule } from 'src/modules/connected-account/connected- WorkspaceDataSourceModule, TypeOrmModule.forFeature([FeatureFlag], 'core'), ConnectedAccountModule, - HealthModule, + MetricsModule, ], providers: [CalendarChannelSyncStatusService], exports: [CalendarChannelSyncStatusService], diff --git a/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts b/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts index 46f0c181d..ac2385d5d 100644 --- a/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts @@ -5,8 +5,8 @@ import { Any } from 'typeorm'; import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; -import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { CalendarChannelSyncStage, @@ -24,7 +24,7 @@ export class CalendarChannelSyncStatusService { @InjectCacheStorage(CacheStorageNamespace.ModuleCalendar) private readonly cacheStorage: CacheStorageService, private readonly accountsToReconnectService: AccountsToReconnectService, - private readonly healthCacheService: HealthCacheService, + private readonly metricsService: MetricsService, ) {} public async scheduleFullCalendarEventListFetch( @@ -179,11 +179,10 @@ export class CalendarChannelSyncStatusService { await this.schedulePartialCalendarEventListFetch(calendarChannelIds); - await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache( - HealthCounterCacheKeys.CalendarEventSyncJobByStatus, - CalendarChannelSyncStatus.ACTIVE, - calendarChannelIds, - ); + await this.metricsService.batchIncrementCounter({ + key: MetricsKeys.CalendarEventSyncJobActive, + eventIds: calendarChannelIds, + }); } public async markAsFailedUnknownAndFlushCalendarEventsToImport( @@ -210,11 +209,10 @@ export class CalendarChannelSyncStatusService { syncStage: CalendarChannelSyncStage.FAILED, }); - await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache( - HealthCounterCacheKeys.CalendarEventSyncJobByStatus, - CalendarChannelSyncStatus.FAILED_UNKNOWN, - calendarChannelIds, - ); + await this.metricsService.batchIncrementCounter({ + key: MetricsKeys.CalendarEventSyncJobFailedUnknown, + eventIds: calendarChannelIds, + }); } public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport( @@ -266,11 +264,10 @@ export class CalendarChannelSyncStatusService { workspaceId, ); - await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache( - HealthCounterCacheKeys.CalendarEventSyncJobByStatus, - CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS, - calendarChannelIds, - ); + await this.metricsService.batchIncrementCounter({ + key: MetricsKeys.CalendarEventSyncJobFailedInsufficientPermissions, + eventIds: calendarChannelIds, + }); } private async addToAccountsToReconnect( diff --git a/packages/twenty-server/src/modules/messaging/common/messaging-common.module.ts b/packages/twenty-server/src/modules/messaging/common/messaging-common.module.ts index 3bd2111be..cf97a07a6 100644 --- a/packages/twenty-server/src/modules/messaging/common/messaging-common.module.ts +++ b/packages/twenty-server/src/modules/messaging/common/messaging-common.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; -import { HealthModule } from 'src/engine/core-modules/health/health.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; @@ -12,7 +12,7 @@ import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/se WorkspaceDataSourceModule, TypeOrmModule.forFeature([FeatureFlag], 'core'), ConnectedAccountModule, - HealthModule, + MetricsModule, ], providers: [MessageChannelSyncStatusService], exports: [MessageChannelSyncStatusService], diff --git a/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts b/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts index 0a06eed84..67acbbdf3 100644 --- a/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts @@ -5,8 +5,8 @@ import { Any } from 'typeorm'; import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; -import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; @@ -24,7 +24,7 @@ export class MessageChannelSyncStatusService { private readonly cacheStorage: CacheStorageService, private readonly twentyORMManager: TwentyORMManager, private readonly accountsToReconnectService: AccountsToReconnectService, - private readonly healthCacheService: HealthCacheService, + private readonly metricsService: MetricsService, ) {} public async scheduleFullMessageListFetch(messageChannelIds: string[]) { @@ -152,11 +152,10 @@ export class MessageChannelSyncStatusService { syncedAt: new Date().toISOString(), }); - await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache( - HealthCounterCacheKeys.MessageChannelSyncJobByStatus, - MessageChannelSyncStatus.ACTIVE, - messageChannelIds, - ); + await this.metricsService.batchIncrementCounter({ + key: MetricsKeys.MessageChannelSyncJobActive, + eventIds: messageChannelIds, + }); } public async markAsMessagesImportOngoing(messageChannelIds: string[]) { @@ -199,11 +198,10 @@ export class MessageChannelSyncStatusService { syncStatus: MessageChannelSyncStatus.FAILED_UNKNOWN, }); - await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache( - HealthCounterCacheKeys.MessageChannelSyncJobByStatus, - MessageChannelSyncStatus.FAILED_UNKNOWN, - messageChannelIds, - ); + await this.metricsService.batchIncrementCounter({ + key: MetricsKeys.MessageChannelSyncJobFailedUnknown, + eventIds: messageChannelIds, + }); } public async markAsFailedInsufficientPermissionsAndFlushMessagesToImport( @@ -230,11 +228,10 @@ export class MessageChannelSyncStatusService { syncStatus: MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS, }); - await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache( - HealthCounterCacheKeys.MessageChannelSyncJobByStatus, - MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS, - messageChannelIds, - ); + await this.metricsService.batchIncrementCounter({ + key: MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions, + eventIds: messageChannelIds, + }); const connectedAccountRepository = await this.twentyORMManager.getRepository( diff --git a/packages/twenty-server/src/queue-worker/queue-worker.ts b/packages/twenty-server/src/queue-worker/queue-worker.ts index 157e098a2..ec3396d70 100644 --- a/packages/twenty-server/src/queue-worker/queue-worker.ts +++ b/packages/twenty-server/src/queue-worker/queue-worker.ts @@ -3,8 +3,8 @@ import { NestFactory } from '@nestjs/core'; import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; import { shouldFilterException } from 'src/engine/utils/global-exception-handler.util'; -import { QueueWorkerModule } from 'src/queue-worker/queue-worker.module'; import 'src/instrument'; +import { QueueWorkerModule } from 'src/queue-worker/queue-worker.module'; async function bootstrap() { let exceptionHandlerService: ExceptionHandlerService | undefined; diff --git a/packages/twenty-server/src/utils/parse-array-env-var.ts b/packages/twenty-server/src/utils/parse-array-env-var.ts new file mode 100644 index 000000000..9ae092e93 --- /dev/null +++ b/packages/twenty-server/src/utils/parse-array-env-var.ts @@ -0,0 +1,13 @@ +export const parseArrayEnvVar = ( + envVar: string | undefined, + expectedValues: T[], + defaultValues: T[], +): T[] => { + if (!envVar) return defaultValues; + + const values = envVar + .split(',') + .filter((item) => expectedValues.includes(item as T)) as T[]; + + return values.length > 0 ? values : defaultValues; +}; diff --git a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx index 232b6ad68..fb47b4995 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx @@ -279,7 +279,7 @@ yarn command:prod cron:calendar:ongoing-stale ['SERVERLESS_LAMBDA_SECRET_ACCESS_KEY', '', 'Optional depending on the authentication method'], ]}> -### Logging +### Logging and Observability diff --git a/yarn.lock b/yarn.lock index 25929632e..185298fa9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11885,6 +11885,15 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/api-logs@npm:0.200.0": + version: 0.200.0 + resolution: "@opentelemetry/api-logs@npm:0.200.0" + dependencies: + "@opentelemetry/api": "npm:^1.3.0" + checksum: 10c0/c6bc3cfba35c69411f294519d93d0ff9f603517030d1162839ee42ac22ed1b0235edaf71d00cabc40125f813d8b4dc830d14315afcebcef138c1df560eaa5c91 + languageName: node + linkType: hard + "@opentelemetry/api-logs@npm:0.52.1": version: 0.52.1 resolution: "@opentelemetry/api-logs@npm:0.52.1" @@ -11903,7 +11912,7 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.8, @opentelemetry/api@npm:^1.9.0": +"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.8, @opentelemetry/api@npm:^1.9.0": version: 1.9.0 resolution: "@opentelemetry/api@npm:1.9.0" checksum: 10c0/9aae2fe6e8a3a3eeb6c1fdef78e1939cf05a0f37f8a4fae4d6bf2e09eb1e06f966ece85805626e01ba5fab48072b94f19b835449e58b6d26720ee19a58298add @@ -11930,6 +11939,32 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/core@npm:2.0.0": + version: 2.0.0 + resolution: "@opentelemetry/core@npm:2.0.0" + dependencies: + "@opentelemetry/semantic-conventions": "npm:^1.29.0" + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.10.0" + checksum: 10c0/d2cc6d8a955305b9de15cc36135e5d5b0f0405fead8bbd4de51433f2d05369af0a3bcb2c6fe7fe6d9e61b0db782511bcadc5d93ed906027d4c00d5c2e3575a24 + languageName: node + linkType: hard + +"@opentelemetry/exporter-metrics-otlp-http@npm:^0.200.0": + version: 0.200.0 + resolution: "@opentelemetry/exporter-metrics-otlp-http@npm:0.200.0" + dependencies: + "@opentelemetry/core": "npm:2.0.0" + "@opentelemetry/otlp-exporter-base": "npm:0.200.0" + "@opentelemetry/otlp-transformer": "npm:0.200.0" + "@opentelemetry/resources": "npm:2.0.0" + "@opentelemetry/sdk-metrics": "npm:2.0.0" + peerDependencies: + "@opentelemetry/api": ^1.3.0 + checksum: 10c0/b94e99a481d3a156ed28f23ea9a72b594613081892ef889fd31be686d32daaa299e961bf173cdc3dcc5235a59f70f6437dd4d21c0256d638c669999c7912c20f + languageName: node + linkType: hard + "@opentelemetry/instrumentation-connect@npm:0.39.0": version: 0.39.0 resolution: "@opentelemetry/instrumentation-connect@npm:0.39.0" @@ -12205,6 +12240,35 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/otlp-exporter-base@npm:0.200.0": + version: 0.200.0 + resolution: "@opentelemetry/otlp-exporter-base@npm:0.200.0" + dependencies: + "@opentelemetry/core": "npm:2.0.0" + "@opentelemetry/otlp-transformer": "npm:0.200.0" + peerDependencies: + "@opentelemetry/api": ^1.3.0 + checksum: 10c0/3283c12bffc3156a41d9c16c097966e8418781a1d779250334f3d5b4f864be1aeac69fecfdf489abc95578dc36098dc0e026e5a48eb19ee170d72ef89b94f0e9 + languageName: node + linkType: hard + +"@opentelemetry/otlp-transformer@npm:0.200.0": + version: 0.200.0 + resolution: "@opentelemetry/otlp-transformer@npm:0.200.0" + dependencies: + "@opentelemetry/api-logs": "npm:0.200.0" + "@opentelemetry/core": "npm:2.0.0" + "@opentelemetry/resources": "npm:2.0.0" + "@opentelemetry/sdk-logs": "npm:0.200.0" + "@opentelemetry/sdk-metrics": "npm:2.0.0" + "@opentelemetry/sdk-trace-base": "npm:2.0.0" + protobufjs: "npm:^7.3.0" + peerDependencies: + "@opentelemetry/api": ^1.3.0 + checksum: 10c0/4f5383fad48c62e17824df91f6944b0376cb17f7b132b11d62fa5cf46747f224c980960209c85669b6e341a131f94586c6ad52bc1a6d2fb8d5295e23b460600c + languageName: node + linkType: hard + "@opentelemetry/redis-common@npm:^0.36.2": version: 0.36.2 resolution: "@opentelemetry/redis-common@npm:0.36.2" @@ -12224,6 +12288,43 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/resources@npm:2.0.0": + version: 2.0.0 + resolution: "@opentelemetry/resources@npm:2.0.0" + dependencies: + "@opentelemetry/core": "npm:2.0.0" + "@opentelemetry/semantic-conventions": "npm:^1.29.0" + peerDependencies: + "@opentelemetry/api": ">=1.3.0 <1.10.0" + checksum: 10c0/2f331ff8268ef7168e8f24312fd7505900693c0ea302f6025937e94c157b8173ee54f5d5a737c06b956da721aa63443ac520f530cade880ef3cd40a2a25c702c + languageName: node + linkType: hard + +"@opentelemetry/sdk-logs@npm:0.200.0": + version: 0.200.0 + resolution: "@opentelemetry/sdk-logs@npm:0.200.0" + dependencies: + "@opentelemetry/api-logs": "npm:0.200.0" + "@opentelemetry/core": "npm:2.0.0" + "@opentelemetry/resources": "npm:2.0.0" + peerDependencies: + "@opentelemetry/api": ">=1.4.0 <1.10.0" + checksum: 10c0/031dc40dd012fad102e5c8c0c9bdbbce051dbc7fcc2e05e003f959aeb34d252dc3595b353ea2a9f900ff40f45d19cb4c8f7ab95a9faa01391f6b415c7780c786 + languageName: node + linkType: hard + +"@opentelemetry/sdk-metrics@npm:2.0.0, @opentelemetry/sdk-metrics@npm:^2.0.0": + version: 2.0.0 + resolution: "@opentelemetry/sdk-metrics@npm:2.0.0" + dependencies: + "@opentelemetry/core": "npm:2.0.0" + "@opentelemetry/resources": "npm:2.0.0" + peerDependencies: + "@opentelemetry/api": ">=1.9.0 <1.10.0" + checksum: 10c0/9a3c87738671f29a496a39d65b3ab0829b52d0f31c0be662ea575a8f77bc5444044fd01513c891abdff6bf6344a08730e18f79253a85e68962669f3e1fa12e72 + languageName: node + linkType: hard + "@opentelemetry/sdk-metrics@npm:^1.9.1": version: 1.26.0 resolution: "@opentelemetry/sdk-metrics@npm:1.26.0" @@ -12236,6 +12337,19 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/sdk-trace-base@npm:2.0.0": + version: 2.0.0 + resolution: "@opentelemetry/sdk-trace-base@npm:2.0.0" + dependencies: + "@opentelemetry/core": "npm:2.0.0" + "@opentelemetry/resources": "npm:2.0.0" + "@opentelemetry/semantic-conventions": "npm:^1.29.0" + peerDependencies: + "@opentelemetry/api": ">=1.3.0 <1.10.0" + checksum: 10c0/c63cc052741e4cc01d084c883e24a1c0792f081a242e14e5cf526d5a3d96bac5974006fa0d8f902bd04f34ed9ce95a0d0f01b7fdb37fcc813cea9f818f2b8f43 + languageName: node + linkType: hard + "@opentelemetry/sdk-trace-base@npm:^1.22, @opentelemetry/sdk-trace-base@npm:^1.26.0": version: 1.26.0 resolution: "@opentelemetry/sdk-trace-base@npm:1.26.0" @@ -12256,6 +12370,13 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/semantic-conventions@npm:^1.29.0": + version: 1.30.0 + resolution: "@opentelemetry/semantic-conventions@npm:1.30.0" + checksum: 10c0/0bf99552e3b4b7e8b7eb504b678d52f59c6f259df88e740a2011a0d858e523d36fee86047ae1b7f45849c77f00f970c3059ba58e0a06a7d47d6f01dbe8c455bd + languageName: node + linkType: hard + "@opentelemetry/sql-common@npm:^0.40.1": version: 0.40.1 resolution: "@opentelemetry/sql-common@npm:0.40.1" @@ -22204,6 +22325,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:>=13.7.0": + version: 22.13.14 + resolution: "@types/node@npm:22.13.14" + dependencies: + undici-types: "npm:~6.20.0" + checksum: 10c0/fa2ab5b8277bfbcc86c42e46a3ea9871b0d559894cc9d955685d17178c9499f0b1bf03d1d1ea8d92ef2dda818988f4035acb8abf9dc15423a998fa56173ab804 + languageName: node + linkType: hard + "@types/node@npm:^10.1.0": version: 10.17.60 resolution: "@types/node@npm:10.17.60" @@ -40434,6 +40564,13 @@ __metadata: languageName: node linkType: hard +"long@npm:^5.0.0": + version: 5.3.1 + resolution: "long@npm:5.3.1" + checksum: 10c0/8726994c6359bb7162fb94563e14c3f9c0f0eeafd90ec654738f4f144a5705756d36a873c442f172ee2a4b51e08d14ab99765b49aa1fb994c5ba7fe12057bca2 + languageName: node + linkType: hard + "longest-streak@npm:^2.0.0": version: 2.0.4 resolution: "longest-streak@npm:2.0.4" @@ -47139,6 +47276,26 @@ __metadata: languageName: node linkType: hard +"protobufjs@npm:^7.3.0": + version: 7.4.0 + resolution: "protobufjs@npm:7.4.0" + dependencies: + "@protobufjs/aspromise": "npm:^1.1.2" + "@protobufjs/base64": "npm:^1.1.2" + "@protobufjs/codegen": "npm:^2.0.4" + "@protobufjs/eventemitter": "npm:^1.1.0" + "@protobufjs/fetch": "npm:^1.1.0" + "@protobufjs/float": "npm:^1.0.2" + "@protobufjs/inquire": "npm:^1.1.0" + "@protobufjs/path": "npm:^1.1.2" + "@protobufjs/pool": "npm:^1.1.0" + "@protobufjs/utf8": "npm:^1.1.0" + "@types/node": "npm:>=13.7.0" + long: "npm:^5.0.0" + checksum: 10c0/a5460a63fe596523b9a067cbce39a6b310d1a71750fda261f076535662aada97c24450e18c5bc98a27784f70500615904ff1227e1742183509f0db4fdede669b + languageName: node + linkType: hard + "proxy-addr@npm:~2.0.7": version: 2.0.7 resolution: "proxy-addr@npm:2.0.7" @@ -53021,6 +53178,9 @@ __metadata: "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch" "@node-saml/passport-saml": "npm:^5.0.0" "@nx/js": "npm:18.3.3" + "@opentelemetry/api": "npm:^1.9.0" + "@opentelemetry/exporter-metrics-otlp-http": "npm:^0.200.0" + "@opentelemetry/sdk-metrics": "npm:^2.0.0" "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch" "@revertdotdev/revert-react": "npm:^0.0.21" "@sentry/nestjs": "npm:^8.30.0"