refactor + new account sync metrics + isolating health status inside folder admin-panel > health-status (#10314)
closes https://github.com/twentyhq/core-team-issues/issues/444 https://github.com/twentyhq/core-team-issues/issues/443 https://github.com/twentyhq/core-team-issues/issues/442
This commit is contained in:
@ -1,11 +1,13 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service';
|
||||
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
||||
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
|
||||
@ -14,44 +16,21 @@ describe('AdminPanelHealthService', () => {
|
||||
let databaseHealth: jest.Mocked<DatabaseHealthIndicator>;
|
||||
let redisHealth: jest.Mocked<RedisHealthIndicator>;
|
||||
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
|
||||
let messageSyncHealth: jest.Mocked<MessageSyncHealthIndicator>;
|
||||
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
|
||||
|
||||
beforeEach(async () => {
|
||||
databaseHealth = {
|
||||
isHealthy: jest.fn(),
|
||||
} as any;
|
||||
|
||||
redisHealth = {
|
||||
isHealthy: jest.fn(),
|
||||
} as any;
|
||||
|
||||
workerHealth = {
|
||||
isHealthy: jest.fn(),
|
||||
} as any;
|
||||
|
||||
messageSyncHealth = {
|
||||
isHealthy: jest.fn(),
|
||||
} as any;
|
||||
databaseHealth = { isHealthy: jest.fn() } as any;
|
||||
redisHealth = { isHealthy: jest.fn() } as any;
|
||||
workerHealth = { isHealthy: jest.fn() } as any;
|
||||
connectedAccountHealth = { isHealthy: jest.fn() } as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
AdminPanelHealthService,
|
||||
{
|
||||
provide: DatabaseHealthIndicator,
|
||||
useValue: databaseHealth,
|
||||
},
|
||||
{
|
||||
provide: RedisHealthIndicator,
|
||||
useValue: redisHealth,
|
||||
},
|
||||
{
|
||||
provide: WorkerHealthIndicator,
|
||||
useValue: workerHealth,
|
||||
},
|
||||
{
|
||||
provide: MessageSyncHealthIndicator,
|
||||
useValue: messageSyncHealth,
|
||||
},
|
||||
{ provide: DatabaseHealthIndicator, useValue: databaseHealth },
|
||||
{ provide: RedisHealthIndicator, useValue: redisHealth },
|
||||
{ provide: WorkerHealthIndicator, useValue: workerHealth },
|
||||
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
@ -62,132 +41,244 @@ describe('AdminPanelHealthService', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it('should transform health check response to SystemHealth format', async () => {
|
||||
databaseHealth.isHealthy.mockResolvedValue({
|
||||
database: {
|
||||
status: 'up',
|
||||
details: 'Database is healthy',
|
||||
},
|
||||
});
|
||||
redisHealth.isHealthy.mockResolvedValue({
|
||||
redis: {
|
||||
status: 'up',
|
||||
details: 'Redis is connected',
|
||||
},
|
||||
});
|
||||
workerHealth.isHealthy.mockResolvedValue({
|
||||
worker: {
|
||||
status: 'up',
|
||||
queues: [
|
||||
{
|
||||
name: 'test',
|
||||
workers: 1,
|
||||
metrics: {
|
||||
active: 1,
|
||||
completed: 0,
|
||||
delayed: 4,
|
||||
failed: 3,
|
||||
waiting: 0,
|
||||
prioritized: 0,
|
||||
describe('getSystemHealthStatus', () => {
|
||||
it('should transform health check response to SystemHealth format', async () => {
|
||||
databaseHealth.isHealthy.mockResolvedValue({
|
||||
database: { status: 'up', details: 'Database is healthy' },
|
||||
});
|
||||
redisHealth.isHealthy.mockResolvedValue({
|
||||
redis: { status: 'up', details: 'Redis is connected' },
|
||||
});
|
||||
workerHealth.isHealthy.mockResolvedValue({
|
||||
worker: {
|
||||
status: 'up',
|
||||
queues: [
|
||||
{
|
||||
queueName: 'test',
|
||||
workers: 1,
|
||||
metrics: {
|
||||
active: 1,
|
||||
completed: 0,
|
||||
delayed: 4,
|
||||
failed: 3,
|
||||
waiting: 0,
|
||||
prioritized: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
connectedAccountHealth.isHealthy.mockResolvedValue({
|
||||
connectedAccount: {
|
||||
status: 'up',
|
||||
details: 'Account sync is operational',
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
|
||||
const expected: SystemHealth = {
|
||||
services: [
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.database],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.redis],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.worker],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
messageSyncHealth.isHealthy.mockResolvedValue({
|
||||
messageSync: {
|
||||
status: 'up',
|
||||
details: 'Message sync is operational',
|
||||
},
|
||||
};
|
||||
|
||||
expect(result).toStrictEqual(expected);
|
||||
});
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
it('should handle mixed health statuses', async () => {
|
||||
databaseHealth.isHealthy.mockResolvedValue({
|
||||
database: { status: 'up' },
|
||||
});
|
||||
redisHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
workerHealth.isHealthy.mockResolvedValue({
|
||||
worker: { status: 'up', queues: [] },
|
||||
});
|
||||
connectedAccountHealth.isHealthy.mockResolvedValue({
|
||||
connectedAccount: { status: 'up' },
|
||||
});
|
||||
|
||||
const expected: SystemHealth = {
|
||||
database: {
|
||||
const result = await service.getSystemHealthStatus();
|
||||
|
||||
expect(result).toStrictEqual({
|
||||
services: [
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.database],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.redis],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.worker],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle all services down', async () => {
|
||||
databaseHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
|
||||
);
|
||||
redisHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
workerHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS),
|
||||
);
|
||||
connectedAccountHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
|
||||
expect(result).toStrictEqual({
|
||||
services: [
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.database],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.redis],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.worker],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIndicatorHealthStatus', () => {
|
||||
it('should return health status for database indicator', async () => {
|
||||
const details = {
|
||||
version: '15.0',
|
||||
connections: { active: 5, max: 100 },
|
||||
};
|
||||
|
||||
databaseHealth.isHealthy.mockResolvedValue({
|
||||
database: {
|
||||
status: 'up',
|
||||
details,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getIndicatorHealthStatus(
|
||||
HealthIndicatorId.database,
|
||||
);
|
||||
|
||||
expect(result).toStrictEqual({
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.database],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: '"Database is healthy"',
|
||||
details: JSON.stringify(details),
|
||||
queues: undefined,
|
||||
},
|
||||
redis: {
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: '"Redis is connected"',
|
||||
queues: undefined,
|
||||
},
|
||||
worker: {
|
||||
});
|
||||
});
|
||||
|
||||
it('should return health status with queues for worker indicator', async () => {
|
||||
const mockQueues = [
|
||||
{
|
||||
queueName: 'queue1',
|
||||
workers: 2,
|
||||
metrics: {
|
||||
active: 1,
|
||||
completed: 10,
|
||||
delayed: 0,
|
||||
failed: 2,
|
||||
waiting: 5,
|
||||
prioritized: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
queueName: 'queue2',
|
||||
workers: 0,
|
||||
metrics: {
|
||||
active: 0,
|
||||
completed: 5,
|
||||
delayed: 0,
|
||||
failed: 1,
|
||||
waiting: 2,
|
||||
prioritized: 0,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
workerHealth.isHealthy.mockResolvedValue({
|
||||
worker: {
|
||||
status: 'up',
|
||||
queues: mockQueues,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getIndicatorHealthStatus(
|
||||
HealthIndicatorId.worker,
|
||||
);
|
||||
|
||||
expect(result).toStrictEqual({
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.worker],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: undefined,
|
||||
queues: [
|
||||
{
|
||||
name: 'test',
|
||||
workers: 1,
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
metrics: {
|
||||
active: 1,
|
||||
completed: 0,
|
||||
delayed: 4,
|
||||
failed: 3,
|
||||
waiting: 0,
|
||||
prioritized: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
messageSync: {
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: '"Message sync is operational"',
|
||||
queues: undefined,
|
||||
},
|
||||
};
|
||||
|
||||
expect(result).toStrictEqual(expected);
|
||||
});
|
||||
|
||||
it('should handle mixed health statuses', async () => {
|
||||
databaseHealth.isHealthy.mockResolvedValue({
|
||||
database: { status: 'up' },
|
||||
});
|
||||
redisHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
workerHealth.isHealthy.mockResolvedValue({
|
||||
worker: { status: 'up', queues: [] },
|
||||
});
|
||||
messageSyncHealth.isHealthy.mockResolvedValue({
|
||||
messageSync: { status: 'up' },
|
||||
queues: mockQueues.map((queue) => ({
|
||||
...queue,
|
||||
id: `worker-${queue.queueName}`,
|
||||
status:
|
||||
queue.workers > 0
|
||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
})),
|
||||
});
|
||||
});
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
it('should handle failed indicator health check', async () => {
|
||||
redisHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
database: { status: AdminPanelHealthServiceStatus.OPERATIONAL },
|
||||
redis: { status: AdminPanelHealthServiceStatus.OUTAGE },
|
||||
worker: { status: AdminPanelHealthServiceStatus.OPERATIONAL },
|
||||
messageSync: { status: AdminPanelHealthServiceStatus.OPERATIONAL },
|
||||
const result = await service.getIndicatorHealthStatus(
|
||||
HealthIndicatorId.redis,
|
||||
);
|
||||
|
||||
expect(result).toStrictEqual({
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.redis],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
details: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle all services down', async () => {
|
||||
databaseHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
|
||||
);
|
||||
redisHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
workerHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS),
|
||||
);
|
||||
messageSyncHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
|
||||
expect(result).toMatchObject({
|
||||
database: { status: AdminPanelHealthServiceStatus.OUTAGE },
|
||||
redis: { status: AdminPanelHealthServiceStatus.OUTAGE },
|
||||
worker: { status: AdminPanelHealthServiceStatus.OUTAGE },
|
||||
messageSync: { status: AdminPanelHealthServiceStatus.OUTAGE },
|
||||
it('should throw error for invalid indicator', async () => {
|
||||
await expect(
|
||||
// @ts-expect-error Testing invalid input
|
||||
service.getIndicatorHealthStatus('invalid'),
|
||||
).rejects.toThrow('Health indicator not found: invalid');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,12 +1,13 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { HealthIndicatorResult } from '@nestjs/terminus';
|
||||
import { HealthIndicatorResult, HealthIndicatorStatus } from '@nestjs/terminus';
|
||||
|
||||
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
||||
import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto';
|
||||
import { AdminPanelIndicatorHealthStatusInputEnum } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input';
|
||||
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
|
||||
@ -16,57 +17,84 @@ export class AdminPanelHealthService {
|
||||
private readonly databaseHealth: DatabaseHealthIndicator,
|
||||
private readonly redisHealth: RedisHealthIndicator,
|
||||
private readonly workerHealth: WorkerHealthIndicator,
|
||||
private readonly messageSyncHealth: MessageSyncHealthIndicator,
|
||||
private readonly connectedAccountHealth: ConnectedAccountHealth,
|
||||
) {}
|
||||
|
||||
private readonly healthIndicators = {
|
||||
database: this.databaseHealth,
|
||||
redis: this.redisHealth,
|
||||
worker: this.workerHealth,
|
||||
messageSync: this.messageSyncHealth,
|
||||
[HealthIndicatorId.database]: this.databaseHealth,
|
||||
[HealthIndicatorId.redis]: this.redisHealth,
|
||||
[HealthIndicatorId.worker]: this.workerHealth,
|
||||
[HealthIndicatorId.connectedAccount]: this.connectedAccountHealth,
|
||||
};
|
||||
|
||||
private transformStatus(status: HealthIndicatorStatus) {
|
||||
return status === 'up'
|
||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
: AdminPanelHealthServiceStatus.OUTAGE;
|
||||
}
|
||||
|
||||
private transformServiceDetails(details: any) {
|
||||
if (!details) return details;
|
||||
|
||||
if (details.messageSync) {
|
||||
details.messageSync.status = this.transformStatus(
|
||||
details.messageSync.status,
|
||||
);
|
||||
}
|
||||
if (details.calendarSync) {
|
||||
details.calendarSync.status = this.transformStatus(
|
||||
details.calendarSync.status,
|
||||
);
|
||||
}
|
||||
|
||||
return details;
|
||||
}
|
||||
|
||||
private getServiceStatus(
|
||||
result: PromiseSettledResult<HealthIndicatorResult>,
|
||||
indicatorId: HealthIndicatorId,
|
||||
) {
|
||||
if (result.status === 'fulfilled') {
|
||||
const key = Object.keys(result.value)[0];
|
||||
const serviceResult = result.value[key];
|
||||
const details = serviceResult.details;
|
||||
const details = this.transformServiceDetails(serviceResult.details);
|
||||
const indicator = HEALTH_INDICATORS[indicatorId];
|
||||
|
||||
return {
|
||||
status:
|
||||
serviceResult.status === 'up'
|
||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
id: indicatorId,
|
||||
label: indicator.label,
|
||||
description: indicator.description,
|
||||
status: this.transformStatus(serviceResult.status),
|
||||
details: details ? JSON.stringify(details) : undefined,
|
||||
queues: serviceResult.queues,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...HEALTH_INDICATORS[indicatorId],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
details: result.reason?.message,
|
||||
details: result.reason?.message?.toString(),
|
||||
};
|
||||
}
|
||||
|
||||
async getIndicatorHealthStatus(
|
||||
indicatorName: AdminPanelIndicatorHealthStatusInputEnum,
|
||||
indicatorId: HealthIndicatorId,
|
||||
): Promise<AdminPanelHealthServiceData> {
|
||||
const healthIndicator = this.healthIndicators[indicatorName];
|
||||
const healthIndicator = this.healthIndicators[indicatorId];
|
||||
|
||||
if (!healthIndicator) {
|
||||
throw new Error(`Health indicator not found: ${indicatorName}`);
|
||||
throw new Error(`Health indicator not found: ${indicatorId}`);
|
||||
}
|
||||
|
||||
const result = await Promise.allSettled([healthIndicator.isHealthy()]);
|
||||
const indicatorStatus = this.getServiceStatus(result[0]);
|
||||
const indicatorStatus = this.getServiceStatus(result[0], indicatorId);
|
||||
|
||||
if (indicatorName === 'worker') {
|
||||
if (indicatorId === HealthIndicatorId.worker) {
|
||||
return {
|
||||
...indicatorStatus,
|
||||
queues: (indicatorStatus?.queues ?? []).map((queue) => ({
|
||||
...queue,
|
||||
id: `${indicatorId}-${queue.queueName}`,
|
||||
status:
|
||||
queue.workers > 0
|
||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
@ -79,30 +107,41 @@ export class AdminPanelHealthService {
|
||||
}
|
||||
|
||||
async getSystemHealthStatus(): Promise<SystemHealth> {
|
||||
const [databaseResult, redisResult, workerResult, messageSyncResult] =
|
||||
const [databaseResult, redisResult, workerResult, accountSyncResult] =
|
||||
await Promise.allSettled([
|
||||
this.databaseHealth.isHealthy(),
|
||||
this.redisHealth.isHealthy(),
|
||||
this.workerHealth.isHealthy(),
|
||||
this.messageSyncHealth.isHealthy(),
|
||||
this.connectedAccountHealth.isHealthy(),
|
||||
]);
|
||||
|
||||
const workerStatus = this.getServiceStatus(workerResult);
|
||||
|
||||
return {
|
||||
database: this.getServiceStatus(databaseResult),
|
||||
redis: this.getServiceStatus(redisResult),
|
||||
worker: {
|
||||
...workerStatus,
|
||||
queues: (workerStatus?.queues ?? []).map((queue) => ({
|
||||
...queue,
|
||||
status:
|
||||
queue.workers > 0
|
||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
})),
|
||||
},
|
||||
messageSync: this.getServiceStatus(messageSyncResult),
|
||||
services: [
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.database],
|
||||
status: this.getServiceStatus(
|
||||
databaseResult,
|
||||
HealthIndicatorId.database,
|
||||
).status,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.redis],
|
||||
status: this.getServiceStatus(redisResult, HealthIndicatorId.redis)
|
||||
.status,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.worker],
|
||||
status: this.getServiceStatus(workerResult, HealthIndicatorId.worker)
|
||||
.status,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
|
||||
status: this.getServiceStatus(
|
||||
accountSyncResult,
|
||||
HealthIndicatorId.connectedAccount,
|
||||
).status,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -11,12 +11,12 @@ import { UpdateWorkspaceFeatureFlagInput } from 'src/engine/core-modules/admin-p
|
||||
import { UserLookup } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.entity';
|
||||
import { UserLookupInput } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.input';
|
||||
import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-graphql-api-exception.filter';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { ImpersonateGuard } from 'src/engine/guards/impersonate-guard';
|
||||
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
|
||||
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
||||
|
||||
import { AdminPanelHealthServiceData } from './dtos/admin-panel-health-service-data.dto';
|
||||
import { AdminPanelIndicatorHealthStatusInputEnum } from './dtos/admin-panel-indicator-health-status.input';
|
||||
|
||||
@Resolver()
|
||||
@UseFilters(AuthGraphqlApiExceptionFilter)
|
||||
@ -70,11 +70,11 @@ export class AdminPanelResolver {
|
||||
|
||||
@Query(() => AdminPanelHealthServiceData)
|
||||
async getIndicatorHealthStatus(
|
||||
@Args('indicatorName', {
|
||||
type: () => AdminPanelIndicatorHealthStatusInputEnum,
|
||||
@Args('indicatorId', {
|
||||
type: () => HealthIndicatorId,
|
||||
})
|
||||
indicatorName: AdminPanelIndicatorHealthStatusInputEnum,
|
||||
indicatorId: HealthIndicatorId,
|
||||
): Promise<AdminPanelHealthServiceData> {
|
||||
return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorName);
|
||||
return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,31 @@
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
|
||||
type HealthIndicatorInfo = {
|
||||
id: HealthIndicatorId;
|
||||
label: string;
|
||||
description: string;
|
||||
};
|
||||
|
||||
export const HEALTH_INDICATORS: Record<HealthIndicatorId, HealthIndicatorInfo> =
|
||||
{
|
||||
[HealthIndicatorId.database]: {
|
||||
id: HealthIndicatorId.database,
|
||||
label: 'Database Status',
|
||||
description: 'PostgreSQL database connection status',
|
||||
},
|
||||
[HealthIndicatorId.redis]: {
|
||||
id: HealthIndicatorId.redis,
|
||||
label: 'Redis Status',
|
||||
description: 'Redis connection status',
|
||||
},
|
||||
[HealthIndicatorId.worker]: {
|
||||
id: HealthIndicatorId.worker,
|
||||
label: 'Worker Status',
|
||||
description: 'Background job worker status',
|
||||
},
|
||||
[HealthIndicatorId.connectedAccount]: {
|
||||
id: HealthIndicatorId.connectedAccount,
|
||||
label: 'Connected Account Status',
|
||||
description: 'Connected accounts status',
|
||||
},
|
||||
};
|
||||
@ -0,0 +1,8 @@
|
||||
import { Field } from '@nestjs/graphql';
|
||||
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
|
||||
export class HealthIndicatorInput {
|
||||
@Field(() => HealthIndicatorId)
|
||||
indicatorId: HealthIndicatorId;
|
||||
}
|
||||
@ -5,6 +5,15 @@ import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-pan
|
||||
|
||||
@ObjectType()
|
||||
export class AdminPanelHealthServiceData {
|
||||
@Field(() => String)
|
||||
id: string;
|
||||
|
||||
@Field(() => String)
|
||||
label: string;
|
||||
|
||||
@Field(() => String)
|
||||
description: string;
|
||||
|
||||
@Field(() => AdminPanelHealthServiceStatus)
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
import { Field, registerEnumType } from '@nestjs/graphql';
|
||||
|
||||
export enum AdminPanelIndicatorHealthStatusInputEnum {
|
||||
DATABASE = 'database',
|
||||
REDIS = 'redis',
|
||||
WORKER = 'worker',
|
||||
MESSAGE_SYNC = 'messageSync',
|
||||
}
|
||||
|
||||
registerEnumType(AdminPanelIndicatorHealthStatusInputEnum, {
|
||||
name: 'AdminPanelIndicatorHealthStatusInputEnum',
|
||||
});
|
||||
|
||||
export class AdminPanelIndicatorHealthStatusInput {
|
||||
@Field(() => AdminPanelIndicatorHealthStatusInputEnum)
|
||||
indicatorName: AdminPanelIndicatorHealthStatusInputEnum;
|
||||
}
|
||||
@ -5,6 +5,9 @@ import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-q
|
||||
|
||||
@ObjectType()
|
||||
export class AdminPanelWorkerQueueHealth extends WorkerQueueHealth {
|
||||
@Field(() => String)
|
||||
id: string;
|
||||
|
||||
@Field(() => AdminPanelHealthServiceStatus)
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
}
|
||||
|
||||
@ -1,18 +1,22 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
|
||||
@ObjectType()
|
||||
export class SystemHealthService {
|
||||
@Field(() => HealthIndicatorId)
|
||||
id: HealthIndicatorId;
|
||||
|
||||
@Field(() => String)
|
||||
label: string;
|
||||
|
||||
@Field(() => AdminPanelHealthServiceStatus)
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
export class SystemHealth {
|
||||
@Field(() => AdminPanelHealthServiceData)
|
||||
database: AdminPanelHealthServiceData;
|
||||
|
||||
@Field(() => AdminPanelHealthServiceData)
|
||||
redis: AdminPanelHealthServiceData;
|
||||
|
||||
@Field(() => AdminPanelHealthServiceData)
|
||||
worker: AdminPanelHealthServiceData;
|
||||
|
||||
@Field(() => AdminPanelHealthServiceData)
|
||||
messageSync: AdminPanelHealthServiceData;
|
||||
@Field(() => [SystemHealthService])
|
||||
services: SystemHealthService[];
|
||||
}
|
||||
|
||||
@ -9,4 +9,7 @@ export const HEALTH_ERROR_MESSAGES = {
|
||||
MESSAGE_SYNC_TIMEOUT: 'Message sync check timeout',
|
||||
MESSAGE_SYNC_CHECK_FAILED: 'Message sync check failed',
|
||||
MESSAGE_SYNC_HIGH_FAILURE_RATE: 'High failure rate in message sync jobs',
|
||||
CALENDAR_SYNC_TIMEOUT: 'Calendar sync check timeout',
|
||||
CALENDAR_SYNC_CHECK_FAILED: 'Calendar sync check failed',
|
||||
CALENDAR_SYNC_HIGH_FAILURE_RATE: 'High failure rate in calendar sync jobs',
|
||||
} as const;
|
||||
|
||||
@ -0,0 +1 @@
|
||||
export const METRICS_FAILURE_RATE_THRESHOLD = 20;
|
||||
@ -2,6 +2,7 @@ import { HealthCheckService } from '@nestjs/terminus';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
@ -29,6 +30,10 @@ describe('HealthController', () => {
|
||||
provide: WorkerHealthIndicator,
|
||||
useValue: { isHealthy: jest.fn() },
|
||||
},
|
||||
{
|
||||
provide: ConnectedAccountHealth,
|
||||
useValue: { isHealthy: jest.fn() },
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ describe('MetricsController', () => {
|
||||
provide: HealthCacheService,
|
||||
useValue: {
|
||||
getMessageChannelSyncJobByStatusCounter: jest.fn(),
|
||||
getCalendarChannelSyncJobByStatusCounter: jest.fn(),
|
||||
getInvalidCaptchaCounter: jest.fn(),
|
||||
},
|
||||
},
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
import { BadRequestException, Controller, Get, Param } from '@nestjs/common';
|
||||
import { HealthCheck, HealthCheckService } from '@nestjs/terminus';
|
||||
|
||||
import { HealthServiceName } from 'src/engine/core-modules/health/enums/health-service-name.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
@ -13,6 +14,7 @@ export class HealthController {
|
||||
private readonly databaseHealth: DatabaseHealthIndicator,
|
||||
private readonly redisHealth: RedisHealthIndicator,
|
||||
private readonly workerHealth: WorkerHealthIndicator,
|
||||
private readonly connectedAccountHealth: ConnectedAccountHealth,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
@ -23,17 +25,19 @@ export class HealthController {
|
||||
|
||||
@Get('/:serviceName')
|
||||
@HealthCheck()
|
||||
checkService(@Param('serviceName') serviceName: HealthServiceName) {
|
||||
checkService(@Param('indicatorId') indicatorId: HealthIndicatorId) {
|
||||
const checks = {
|
||||
[HealthServiceName.DATABASE]: () => this.databaseHealth.isHealthy(),
|
||||
[HealthServiceName.REDIS]: () => this.redisHealth.isHealthy(),
|
||||
[HealthServiceName.WORKER]: () => this.workerHealth.isHealthy(),
|
||||
[HealthIndicatorId.database]: () => this.databaseHealth.isHealthy(),
|
||||
[HealthIndicatorId.redis]: () => this.redisHealth.isHealthy(),
|
||||
[HealthIndicatorId.worker]: () => this.workerHealth.isHealthy(),
|
||||
[HealthIndicatorId.connectedAccount]: () =>
|
||||
this.connectedAccountHealth.isHealthy(),
|
||||
};
|
||||
|
||||
if (!(serviceName in checks)) {
|
||||
throw new BadRequestException(`Invalid service name: ${serviceName}`);
|
||||
if (!(indicatorId in checks)) {
|
||||
throw new BadRequestException(`Invalid indicatorId: ${indicatorId}`);
|
||||
}
|
||||
|
||||
return this.health.check([checks[serviceName]]);
|
||||
return this.health.check([checks[indicatorId]]);
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,4 +15,9 @@ export class MetricsController {
|
||||
getInvalidCaptchaCounter() {
|
||||
return this.healthCacheService.getInvalidCaptchaCounter();
|
||||
}
|
||||
|
||||
@Get('/calendar-channel-sync-job-by-status-counter')
|
||||
getCalendarChannelSyncJobByStatusCounter() {
|
||||
return this.healthCacheService.getCalendarChannelSyncJobByStatusCounter();
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,12 @@
|
||||
import { registerEnumType } from '@nestjs/graphql';
|
||||
|
||||
export enum HealthIndicatorId {
|
||||
database = 'database',
|
||||
redis = 'redis',
|
||||
worker = 'worker',
|
||||
connectedAccount = 'connectedAccount',
|
||||
}
|
||||
|
||||
registerEnumType(HealthIndicatorId, {
|
||||
name: 'HealthIndicatorId',
|
||||
});
|
||||
@ -1,6 +0,0 @@
|
||||
export enum HealthServiceName {
|
||||
DATABASE = 'database',
|
||||
REDIS = 'redis',
|
||||
WORKER = 'worker',
|
||||
MESSAGE_SYNC = 'messageSync',
|
||||
}
|
||||
@ -4,8 +4,9 @@ import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decora
|
||||
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
|
||||
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
|
||||
import { AccountSyncJobByStatusCounter } from 'src/engine/core-modules/health/types/account-sync-metrics.types';
|
||||
import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type';
|
||||
import { MessageChannelSyncJobByStatusCounter } from 'src/engine/core-modules/health/types/message-sync-metrics.types';
|
||||
import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
|
||||
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
|
||||
@Injectable()
|
||||
@ -48,9 +49,7 @@ export class HealthCacheService {
|
||||
);
|
||||
|
||||
const currentCounter =
|
||||
await this.cacheStorage.get<MessageChannelSyncJobByStatusCounter>(
|
||||
cacheKey,
|
||||
);
|
||||
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(cacheKey);
|
||||
|
||||
const updatedCounter = {
|
||||
...(currentCounter || {}),
|
||||
@ -80,7 +79,7 @@ export class HealthCacheService {
|
||||
|
||||
for (const key of cacheKeys) {
|
||||
const counter =
|
||||
await this.cacheStorage.get<MessageChannelSyncJobByStatusCounter>(key);
|
||||
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(key);
|
||||
|
||||
if (!counter) continue;
|
||||
|
||||
@ -130,4 +129,58 @@ export class HealthCacheService {
|
||||
|
||||
return aggregatedCounter;
|
||||
}
|
||||
|
||||
async incrementCalendarChannelSyncJobByStatusCounter(
|
||||
status: CalendarChannelSyncStatus,
|
||||
increment: number,
|
||||
) {
|
||||
const cacheKey = this.getCacheKeyWithTimestamp(
|
||||
HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
|
||||
);
|
||||
|
||||
const currentCounter =
|
||||
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(cacheKey);
|
||||
|
||||
const updatedCounter = {
|
||||
...(currentCounter || {}),
|
||||
[status]: (currentCounter?.[status] || 0) + increment,
|
||||
};
|
||||
|
||||
return await this.cacheStorage.set(
|
||||
cacheKey,
|
||||
updatedCounter,
|
||||
this.healthCacheTtl,
|
||||
);
|
||||
}
|
||||
|
||||
async getCalendarChannelSyncJobByStatusCounter() {
|
||||
const cacheKeys = this.getLastXMinutesTimestamps(
|
||||
this.healthMonitoringTimeWindowInMinutes,
|
||||
).map((timestamp) =>
|
||||
this.getCacheKeyWithTimestamp(
|
||||
HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
|
||||
timestamp,
|
||||
),
|
||||
);
|
||||
|
||||
const aggregatedCounter = Object.fromEntries(
|
||||
Object.values(CalendarChannelSyncStatus).map((status) => [status, 0]),
|
||||
);
|
||||
|
||||
for (const key of cacheKeys) {
|
||||
const counter =
|
||||
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(key);
|
||||
|
||||
if (!counter) continue;
|
||||
|
||||
for (const [status, count] of Object.entries(counter) as [
|
||||
CalendarChannelSyncStatus,
|
||||
number,
|
||||
][]) {
|
||||
aggregatedCounter[status] += count;
|
||||
}
|
||||
}
|
||||
|
||||
return aggregatedCounter;
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,11 +3,11 @@ import { TerminusModule } from '@nestjs/terminus';
|
||||
|
||||
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
|
||||
import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller';
|
||||
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
|
||||
import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
|
||||
|
||||
import { HealthCacheService } from './health-cache.service';
|
||||
|
||||
import { ConnectedAccountHealth } from './indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from './indicators/database.health';
|
||||
import { RedisHealthIndicator } from './indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from './indicators/worker.health';
|
||||
@ -19,14 +19,14 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
|
||||
DatabaseHealthIndicator,
|
||||
RedisHealthIndicator,
|
||||
WorkerHealthIndicator,
|
||||
MessageSyncHealthIndicator,
|
||||
ConnectedAccountHealth,
|
||||
],
|
||||
exports: [
|
||||
HealthCacheService,
|
||||
DatabaseHealthIndicator,
|
||||
RedisHealthIndicator,
|
||||
WorkerHealthIndicator,
|
||||
MessageSyncHealthIndicator,
|
||||
ConnectedAccountHealth,
|
||||
],
|
||||
})
|
||||
export class HealthModule {}
|
||||
|
||||
@ -0,0 +1,316 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts';
|
||||
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const';
|
||||
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
|
||||
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
|
||||
describe('ConnectedAccountHealth', () => {
|
||||
let service: ConnectedAccountHealth;
|
||||
let healthCacheService: jest.Mocked<HealthCacheService>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
|
||||
beforeEach(async () => {
|
||||
healthCacheService = {
|
||||
getMessageChannelSyncJobByStatusCounter: jest.fn(),
|
||||
getCalendarChannelSyncJobByStatusCounter: jest.fn(),
|
||||
} as any;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockImplementation((key) => ({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
[key]: {
|
||||
status: 'up',
|
||||
details: data.details,
|
||||
},
|
||||
})),
|
||||
down: jest.fn().mockImplementation((data) => ({
|
||||
[key]: {
|
||||
status: 'down',
|
||||
error: data.error,
|
||||
details: data.details,
|
||||
},
|
||||
})),
|
||||
})),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
ConnectedAccountHealth,
|
||||
{
|
||||
provide: HealthCacheService,
|
||||
useValue: healthCacheService,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<ConnectedAccountHealth>(ConnectedAccountHealth);
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe('message sync health', () => {
|
||||
it('should return up status when no message sync jobs are present', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ONGOING]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 0,
|
||||
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 0,
|
||||
},
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 0,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('up');
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('up');
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.totalJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.failedJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.failureRate,
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it(`should return down status when message sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ONGOING]: 1,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
},
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.messageSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.failureRate,
|
||||
).toBe(33.33);
|
||||
});
|
||||
});
|
||||
|
||||
describe('calendar sync health', () => {
|
||||
it('should return up status when no calendar sync jobs are present', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 0,
|
||||
},
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ONGOING]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 0,
|
||||
[CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 0,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('up');
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('up');
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.totalJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.failedJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.failureRate,
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it(`should return down status when calendar sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
},
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ONGOING]: 1,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
[CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.calendarSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.failureRate,
|
||||
).toBe(33.33);
|
||||
});
|
||||
});
|
||||
|
||||
describe('timeout handling', () => {
|
||||
it('should handle message sync timeout', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
},
|
||||
);
|
||||
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.messageSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle calendar sync timeout', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
},
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
);
|
||||
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
|
||||
);
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.calendarSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('combined health check', () => {
|
||||
it('should return combined status with both checks healthy', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 8,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 1,
|
||||
},
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 8,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 1,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('up');
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('up');
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('up');
|
||||
});
|
||||
|
||||
it('should return down status when both syncs fail', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
},
|
||||
);
|
||||
|
||||
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
`${HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE} and ${HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE}`,
|
||||
);
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,137 +0,0 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts';
|
||||
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
|
||||
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
|
||||
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
|
||||
describe('MessageSyncHealthIndicator', () => {
|
||||
let service: MessageSyncHealthIndicator;
|
||||
let healthCacheService: jest.Mocked<HealthCacheService>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
|
||||
beforeEach(async () => {
|
||||
healthCacheService = {
|
||||
getMessageChannelSyncJobByStatusCounter: jest.fn(),
|
||||
} as any;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockReturnValue({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
messageSync: { status: 'up', ...data },
|
||||
})),
|
||||
down: jest.fn().mockImplementation((error) => ({
|
||||
messageSync: { status: 'down', error },
|
||||
})),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
MessageSyncHealthIndicator,
|
||||
{
|
||||
provide: HealthCacheService,
|
||||
useValue: healthCacheService,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<MessageSyncHealthIndicator>(
|
||||
MessageSyncHealthIndicator,
|
||||
);
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return up status when no jobs are present', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ONGOING]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 0,
|
||||
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 0,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.messageSync.status).toBe('up');
|
||||
expect(result.messageSync.details.totalJobs).toBe(0);
|
||||
expect(result.messageSync.details.failedJobs).toBe(0);
|
||||
expect(result.messageSync.details.failureRate).toBe(0);
|
||||
});
|
||||
|
||||
it('should return up status when failure rate is below 20%', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ONGOING]: 2,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 8,
|
||||
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 1,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.messageSync.status).toBe('up');
|
||||
expect(result.messageSync.details.totalJobs).toBe(11);
|
||||
expect(result.messageSync.details.failedJobs).toBe(1);
|
||||
expect(result.messageSync.details.failureRate).toBe(9.09);
|
||||
});
|
||||
|
||||
it('should return down status when failure rate is above 20%', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
|
||||
{
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ONGOING]: 1,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
},
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.messageSync.status).toBe('down');
|
||||
expect(result.messageSync.error.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(result.messageSync.error.details).toBeDefined();
|
||||
expect(result.messageSync.error.details.failureRate).toBe(33.33);
|
||||
});
|
||||
|
||||
it('should timeout after specified duration', async () => {
|
||||
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
);
|
||||
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.messageSync.status).toBe('down');
|
||||
expect(result.messageSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,155 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const';
|
||||
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
|
||||
|
||||
@Injectable()
|
||||
export class ConnectedAccountHealth {
|
||||
constructor(
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
private readonly healthCacheService: HealthCacheService,
|
||||
) {}
|
||||
|
||||
private async checkMessageSyncHealth(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('messageSync');
|
||||
|
||||
try {
|
||||
const counters = await withHealthCheckTimeout(
|
||||
this.healthCacheService.getMessageChannelSyncJobByStatusCounter(),
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
|
||||
const totalJobs = Object.values(counters).reduce(
|
||||
(sum, count) => sum + (count || 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const failedJobs = counters.FAILED_UNKNOWN || 0;
|
||||
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
|
||||
|
||||
const failureRate =
|
||||
totalJobs > 0
|
||||
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
|
||||
: 0;
|
||||
const details = {
|
||||
counters,
|
||||
totalJobs,
|
||||
failedJobs,
|
||||
failureRate,
|
||||
};
|
||||
|
||||
if (totalJobs === 0 || failureRate < METRICS_FAILURE_RATE_THRESHOLD) {
|
||||
return indicator.up({ details });
|
||||
}
|
||||
|
||||
return indicator.down({
|
||||
error: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
details,
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error.message === HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED;
|
||||
|
||||
return indicator.down({
|
||||
error: errorMessage,
|
||||
details: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async checkCalendarSyncHealth(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('calendarSync');
|
||||
|
||||
try {
|
||||
const counters = await withHealthCheckTimeout(
|
||||
this.healthCacheService.getCalendarChannelSyncJobByStatusCounter(),
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
|
||||
);
|
||||
|
||||
const totalJobs = Object.values(counters).reduce(
|
||||
(sum, count) => sum + (count || 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const failedJobs = counters.FAILED_UNKNOWN || 0;
|
||||
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
|
||||
|
||||
const failureRate =
|
||||
totalJobs > 0
|
||||
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
|
||||
: 0;
|
||||
const details = {
|
||||
counters,
|
||||
totalJobs,
|
||||
failedJobs,
|
||||
failureRate,
|
||||
};
|
||||
|
||||
if (totalJobs === 0 || failureRate < METRICS_FAILURE_RATE_THRESHOLD) {
|
||||
return indicator.up({ details });
|
||||
}
|
||||
|
||||
return indicator.down({
|
||||
error: HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
|
||||
details,
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error.message === HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_CHECK_FAILED;
|
||||
|
||||
return indicator.down({
|
||||
error: errorMessage,
|
||||
details: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('connectedAccount');
|
||||
|
||||
const [messageResult, calendarResult] = await Promise.all([
|
||||
this.checkMessageSyncHealth(),
|
||||
this.checkCalendarSyncHealth(),
|
||||
]);
|
||||
|
||||
const isMessageSyncDown = messageResult.messageSync.status === 'down';
|
||||
const isCalendarSyncDown = calendarResult.calendarSync.status === 'down';
|
||||
|
||||
if (isMessageSyncDown || isCalendarSyncDown) {
|
||||
let error: string;
|
||||
|
||||
if (isMessageSyncDown && isCalendarSyncDown) {
|
||||
error = `${messageResult.messageSync.error} and ${calendarResult.calendarSync.error}`;
|
||||
} else if (isMessageSyncDown) {
|
||||
error = messageResult.messageSync.error;
|
||||
} else {
|
||||
error = calendarResult.calendarSync.error;
|
||||
}
|
||||
|
||||
return indicator.down({
|
||||
error,
|
||||
details: {
|
||||
messageSync: messageResult.messageSync,
|
||||
calendarSync: calendarResult.calendarSync,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return indicator.up({
|
||||
details: {
|
||||
messageSync: messageResult.messageSync,
|
||||
calendarSync: calendarResult.calendarSync,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
|
||||
|
||||
@Injectable()
|
||||
export class MessageSyncHealthIndicator {
|
||||
constructor(
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
private readonly healthCacheService: HealthCacheService,
|
||||
) {}
|
||||
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('messageSync');
|
||||
|
||||
try {
|
||||
const counters = await withHealthCheckTimeout(
|
||||
this.healthCacheService.getMessageChannelSyncJobByStatusCounter(),
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
|
||||
const totalJobs = Object.values(counters).reduce(
|
||||
(sum, count) => sum + (count || 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const failedJobs = counters.FAILED_UNKNOWN || 0;
|
||||
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
|
||||
|
||||
const failureRate =
|
||||
totalJobs > 0
|
||||
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
|
||||
: 0;
|
||||
const details = {
|
||||
counters,
|
||||
totalJobs,
|
||||
failedJobs,
|
||||
failureRate,
|
||||
};
|
||||
|
||||
if (totalJobs === 0 || failureRate < 20) {
|
||||
return indicator.up({ details });
|
||||
}
|
||||
|
||||
return indicator.down({
|
||||
error: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
details,
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error.message === HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED;
|
||||
|
||||
return indicator.down(errorMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -71,7 +71,7 @@ export class WorkerHealthIndicator {
|
||||
]);
|
||||
|
||||
queueStatuses.push({
|
||||
name: queueName,
|
||||
queueName: queueName,
|
||||
workers: workers.length,
|
||||
metrics: {
|
||||
failed: failedCount,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
@ObjectType()
|
||||
export class MessageChannelSyncJobByStatusCounter {
|
||||
export class AccountSyncJobByStatusCounter {
|
||||
@Field(() => Number, { nullable: true })
|
||||
NOT_SYNCED?: number;
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
export enum HealthCounterCacheKeys {
|
||||
MessageChannelSyncJobByStatus = 'message-channel-sync-job-by-status',
|
||||
InvalidCaptcha = 'invalid-captcha',
|
||||
CalendarEventSyncJobByStatus = 'calendar-event-sync-job-by-status',
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@ import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-
|
||||
@ObjectType()
|
||||
export class WorkerQueueHealth {
|
||||
@Field(() => String)
|
||||
name: string;
|
||||
queueName: string;
|
||||
|
||||
@Field(() => Number)
|
||||
workers: number;
|
||||
|
||||
Reference in New Issue
Block a user