set up metrics collecting with open telemetry (#11236)

Done :  
- move metrics and health cache services from health module to metrics
module
- refactor metrics counter from specific method to set up from enum keys
- add OpenTelemetry (Otel) instrumentation for metrics
- set up Otel SDK to send metrics to Otel collector

To do later : 
- implement Otel instrumentation for traces + plug Sentry on top
This commit is contained in:
Etienne
2025-03-28 08:45:24 +01:00
committed by GitHub
parent e9e33c4d29
commit 391392dd87
32 changed files with 575 additions and 297 deletions

View File

@ -1,10 +1,27 @@
import * as Sentry from '@sentry/nestjs';
import process from 'process';
import opentelemetry from '@opentelemetry/api';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import {
ConsoleMetricExporter,
MeterProvider,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { NodeEnvironment } from 'src/engine/core-modules/environment/interfaces/node-environment.interface';
import { ExceptionHandlerDriver } from 'src/engine/core-modules/exception-handler/interfaces';
import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type';
import { WorkspaceCacheKeys } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { parseArrayEnvVar } from 'src/utils/parse-array-env-var';
const meterDrivers = parseArrayEnvVar(
process.env.METER_DRIVER,
Object.values(MeterDriver),
[MeterDriver.Console],
);
if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry) {
Sentry.init({
@ -29,3 +46,34 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry) {
debug: process.env.NODE_ENV === NodeEnvironment.development,
});
}
// Meter setup
const meterProvider = new MeterProvider({
readers: [
...(meterDrivers.includes(MeterDriver.Console)
? [
new PeriodicExportingMetricReader({
exporter: new ConsoleMetricExporter(),
exportIntervalMillis: 10000,
}),
]
: []),
...(meterDrivers.includes(MeterDriver.OpenTelemetry)
? [
new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({
url: process.env.OTLP_COLLECTOR_METRICS_ENDPOINT_URL,
}),
exportIntervalMillis: 10000,
}),
]
: []),
],
});
opentelemetry.metrics.setGlobalMeterProvider(meterProvider);
process.on('SIGTERM', async () => {
await meterProvider.shutdown();
});