admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Admin panel: App health check (#10546)

Browse Source 
closes https://github.com/twentyhq/core-team-issues/issues/441
whats new - 
- app health with the proposed format -- update, scratched this format
in favor of a more concise and more intentional health check. Now we
will check app health only based on pendingMigrations
<del>
```
status: 
  system: {
    nodeVersion
  },
  overview: {
    totalWorkspaces,
    criticalWorkspaces,
    workspacesWithPendingMigrations,
    healthDistribution: {
      healthy,
      warning,
      critical,
    }
  },
 problematicWorkspaces: [
    {
      workspaceId,
      severity,
      pendingMigrations,
      issuesSummary: {
        structural,
        data,
        relationship
      }
    }
  ]
  ```
</del>

- errorMessage and details seperation -- before we used to send error in details which made it difficult if we want both error and details on the front -- usecase >>
suppose app health indicator is not healthy but still want to send details
- stateHistory with timestamp -- this is something I introduced, not sure about this. Basically the thought process was to store the LastState of the details, incase of no connection or timeout errors. This is not yet used on the front, just the endpoint. But it could be used on the front too
- name unifying ⁠https://discord.com/channels/1130383047699738754/1346454192776155156
- json tree https://discord.com/channels/1130383047699738754/1346458558048501760
- match figma design https://discord.com/channels/1130383047699738754/1346451659647094815
- fix the collapse/open styles in tables https://discord.com/channels/1130383047699738754/1346452051974160406
- shift eye icon to expanded container https://discord.com/channels/1130383047699738754/1346452282669010987
- use H2Title for title and description of env variables groups https://discord.com/channels/1130383047699738754/1346434955936530452
This commit is contained in:
nitin
2025-03-07 01:49:41 +05:30
committed by GitHub
parent f6314e52fe
commit 3061576302
34 changed files with 808 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts
View File

@ -11,6 +11,7 @@ import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
@ -26,6 +27,7 @@ describe('AdminPanelHealthService', () => {
let redisHealth: jest.Mocked<RedisHealthIndicator>;
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
let appHealth: jest.Mocked<AppHealthIndicator>;
let redisClient: jest.Mocked<RedisClientService>;
let environmentService: jest.Mocked<EnvironmentService>;
let loggerSpy: jest.SpyInstance;
@ -35,6 +37,7 @@ describe('AdminPanelHealthService', () => {
redisHealth = { isHealthy: jest.fn() } as any;
workerHealth = { isHealthy: jest.fn(), getQueueDetails: jest.fn() } as any;
connectedAccountHealth = { isHealthy: jest.fn() } as any;
appHealth = { isHealthy: jest.fn() } as any;
redisClient = {
getClient: jest.fn().mockReturnValue({} as Redis),
} as any;
@ -53,6 +56,7 @@ describe('AdminPanelHealthService', () => {
{ provide: RedisHealthIndicator, useValue: redisHealth },
{ provide: WorkerHealthIndicator, useValue: workerHealth },
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
{ provide: AppHealthIndicator, useValue: appHealth },
{ provide: RedisClientService, useValue: redisClient },
{ provide: EnvironmentService, useValue: environmentService },
],
@ -108,6 +112,32 @@ describe('AdminPanelHealthService', () => {
details: 'Account sync is operational',
},
});
appHealth.isHealthy.mockResolvedValue({
app: {
status: 'up',
details: {
system: {
nodeVersion: '16.0',
},
workspaces: {
totalWorkspaces: 1,
healthStatus: [
{
workspaceId: '1',
summary: {
structuralIssues: 0,
dataIssues: 0,
relationshipIssues: 0,
pendingMigrations: 0,
},
severity: 'healthy',
details: {},
},
],
},
},
},
});
const result = await service.getSystemHealthStatus();
@ -129,6 +159,10 @@ describe('AdminPanelHealthService', () => {
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
],
};
@ -148,6 +182,9 @@ describe('AdminPanelHealthService', () => {
connectedAccountHealth.isHealthy.mockResolvedValue({
connectedAccount: { status: 'up' },
});
appHealth.isHealthy.mockResolvedValue({
app: { status: 'down', details: {} },
});
const result = await service.getSystemHealthStatus();
@ -169,6 +206,10 @@ describe('AdminPanelHealthService', () => {
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
],
});
});
@ -186,6 +227,9 @@ describe('AdminPanelHealthService', () => {
connectedAccountHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED),
);
appHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.APP_HEALTH_CHECK_FAILED),
);
const result = await service.getSystemHealthStatus();
@ -207,6 +251,10 @@ describe('AdminPanelHealthService', () => {
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
],
});
});
@ -233,7 +281,8 @@ describe('AdminPanelHealthService', () => {
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.database],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: JSON.stringify(details),
details: JSON.stringify({ details }),
errorMessage: undefined,
queues: undefined,
});
});
@ -266,7 +315,8 @@ describe('AdminPanelHealthService', () => {
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.worker],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: undefined,
details: JSON.stringify({ queues: mockQueues }),
errorMessage: undefined,
queues: mockQueues.map((queue) => ({
id: `worker-${queue.queueName}`,
queueName: queue.queueName,
@ -290,7 +340,8 @@ describe('AdminPanelHealthService', () => {
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.redis],
status: AdminPanelHealthServiceStatus.OUTAGE,
details: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
details: undefined,
errorMessage: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
});
});

77
packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts
View File

@ -10,6 +10,7 @@ import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-he
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
@ -27,6 +28,7 @@ export class AdminPanelHealthService {
private readonly redisHealth: RedisHealthIndicator,
private readonly workerHealth: WorkerHealthIndicator,
private readonly connectedAccountHealth: ConnectedAccountHealth,
private readonly appHealth: AppHealthIndicator,
private readonly redisClient: RedisClientService,
) {}
@ -35,6 +37,7 @@ export class AdminPanelHealthService {
[HealthIndicatorId.redis]: this.redisHealth,
[HealthIndicatorId.worker]: this.workerHealth,
[HealthIndicatorId.connectedAccount]: this.connectedAccountHealth,
[HealthIndicatorId.app]: this.appHealth,
};
private transformStatus(status: HealthIndicatorStatus) {
@ -46,15 +49,17 @@ export class AdminPanelHealthService {
private transformServiceDetails(details: any) {
if (!details) return details;
if (details.messageSync) {
details.messageSync.status = this.transformStatus(
details.messageSync.status,
);
}
if (details.calendarSync) {
details.calendarSync.status = this.transformStatus(
details.calendarSync.status,
);
if (details.details) {
if (details.details.messageSync) {
details.details.messageSync.status = this.transformStatus(
details.details.messageSync.status,
);
}
if (details.details.calendarSync) {
details.details.calendarSync.status = this.transformStatus(
details.details.calendarSync.status,
);
}
}
return details;
@ -65,17 +70,33 @@ export class AdminPanelHealthService {
indicatorId: HealthIndicatorId,
) {
if (result.status === 'fulfilled') {
const key = Object.keys(result.value)[0];
const keys = Object.keys(result.value);
if (keys.length === 0) {
return {
...HEALTH_INDICATORS[indicatorId],
status: AdminPanelHealthServiceStatus.OUTAGE,
errorMessage: 'No health check result available',
};
}
const key = keys[0];
const serviceResult = result.value[key];
const details = this.transformServiceDetails(serviceResult.details);
const { status, message, ...detailsWithoutStatus } = serviceResult;
const indicator = HEALTH_INDICATORS[indicatorId];
const transformedDetails =
this.transformServiceDetails(detailsWithoutStatus);
return {
id: indicatorId,
label: indicator.label,
description: indicator.description,
status: this.transformStatus(serviceResult.status),
details: details ? JSON.stringify(details) : undefined,
status: this.transformStatus(status),
errorMessage: message,
details:
Object.keys(transformedDetails).length > 0
? JSON.stringify(transformedDetails)
: undefined,
queues: serviceResult.queues,
};
}
@ -83,7 +104,10 @@ export class AdminPanelHealthService {
return {
...HEALTH_INDICATORS[indicatorId],
status: AdminPanelHealthServiceStatus.OUTAGE,
details: result.reason?.message?.toString(),
errorMessage: result.reason?.message?.toString(),
details: result.reason?.details
? JSON.stringify(result.reason.details)
: undefined,
};
}
@ -117,13 +141,19 @@ export class AdminPanelHealthService {
}
async getSystemHealthStatus(): Promise<SystemHealth> {
const [databaseResult, redisResult, workerResult, accountSyncResult] =
await Promise.allSettled([
this.databaseHealth.isHealthy(),
this.redisHealth.isHealthy(),
this.workerHealth.isHealthy(),
this.connectedAccountHealth.isHealthy(),
]);
const [
databaseResult,
redisResult,
workerResult,
accountSyncResult,
appResult,
] = await Promise.allSettled([
this.databaseHealth.isHealthy(),
this.redisHealth.isHealthy(),
this.workerHealth.isHealthy(),
this.connectedAccountHealth.isHealthy(),
this.appHealth.isHealthy(),
]);
return {
services: [
@ -151,6 +181,11 @@ export class AdminPanelHealthService {
HealthIndicatorId.connectedAccount,
).status,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: this.getServiceStatus(appResult, HealthIndicatorId.app)
.status,
},
],
};
}

5
packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators.constants.ts
View File

@ -28,4 +28,9 @@ export const HEALTH_INDICATORS: Record<HealthIndicatorId, HealthIndicatorInfo> =
label: 'Connected Account Status',
description: 'Connected accounts status',
},
[HealthIndicatorId.app]: {
id: HealthIndicatorId.app,
label: 'App Status',
description: 'Workspace metadata migration status check',
},
};

2
packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts
View File

@ -17,6 +17,8 @@ export class AdminPanelHealthServiceData {
@Field(() => AdminPanelHealthServiceStatus)
status: AdminPanelHealthServiceStatus;
@Field(() => String, { nullable: true })
errorMessage?: string;
@Field(() => String, { nullable: true })
details?: string;

4
packages/twenty-server/src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum.ts
View File

@ -1,8 +1,8 @@
import { registerEnumType } from '@nestjs/graphql';
export enum AdminPanelHealthServiceStatus {
OPERATIONAL = 'operational',
OUTAGE = 'outage',
OPERATIONAL = 'OPERATIONAL',
OUTAGE = 'OUTAGE',
}
registerEnumType(AdminPanelHealthServiceStatus, {

1
packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts
View File

@ -12,4 +12,5 @@ export const HEALTH_ERROR_MESSAGES = {
CALENDAR_SYNC_TIMEOUT: 'Calendar sync check timeout',
CALENDAR_SYNC_CHECK_FAILED: 'Calendar sync check failed',
CALENDAR_SYNC_HIGH_FAILURE_RATE: 'High failure rate in calendar sync jobs',
APP_HEALTH_CHECK_FAILED: 'App health check failed',
} as const;

5
packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts
View File

@ -2,6 +2,7 @@ import { HealthCheckService } from '@nestjs/terminus';
import { Test, TestingModule } from '@nestjs/testing';
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
@ -34,6 +35,10 @@ describe('HealthController', () => {
provide: ConnectedAccountHealth,
useValue: { isHealthy: jest.fn() },
},
{
provide: AppHealthIndicator,
useValue: { isHealthy: jest.fn() },
},
],
}).compile();

4
packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts
View File

@ -2,11 +2,11 @@ import { BadRequestException, Controller, Get, Param } from '@nestjs/common';
import { HealthCheck, HealthCheckService } from '@nestjs/terminus';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
@Controller('healthz')
export class HealthController {
constructor(
@ -15,6 +15,7 @@ export class HealthController {
private readonly redisHealth: RedisHealthIndicator,
private readonly workerHealth: WorkerHealthIndicator,
private readonly connectedAccountHealth: ConnectedAccountHealth,
private readonly appHealth: AppHealthIndicator,
) {}
@Get()
@ -32,6 +33,7 @@ export class HealthController {
[HealthIndicatorId.worker]: () => this.workerHealth.isHealthy(),
[HealthIndicatorId.connectedAccount]: () =>
this.connectedAccountHealth.isHealthy(),
[HealthIndicatorId.app]: () => this.appHealth.isHealthy(),
};
if (!(indicatorId in checks)) {

1
packages/twenty-server/src/engine/core-modules/health/enums/health-indicator-id.enum.ts
View File

@ -5,6 +5,7 @@ export enum HealthIndicatorId {
redis = 'redis',
worker = 'worker',
connectedAccount = 'connectedAccount',
app = 'app',
}
registerEnumType(HealthIndicatorId, {

14
packages/twenty-server/src/engine/core-modules/health/health.module.ts
View File

@ -3,7 +3,11 @@ import { TerminusModule } from '@nestjs/terminus';
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { HealthCacheService } from './health-cache.service';
@ -12,7 +16,13 @@ import { DatabaseHealthIndicator } from './indicators/database.health';
import { RedisHealthIndicator } from './indicators/redis.health';
import { WorkerHealthIndicator } from './indicators/worker.health';
@Module({
imports: [TerminusModule, RedisClientModule],
imports: [
TerminusModule,
RedisClientModule,
WorkspaceHealthModule,
ObjectMetadataModule,
WorkspaceMigrationModule,
],
controllers: [HealthController, MetricsController],
providers: [
HealthCacheService,
@ -20,6 +30,7 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
RedisHealthIndicator,
WorkerHealthIndicator,
ConnectedAccountHealth,
AppHealthIndicator,
],
exports: [
HealthCacheService,
@ -27,6 +38,7 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
RedisHealthIndicator,
WorkerHealthIndicator,
ConnectedAccountHealth,
AppHealthIndicator,
],
})
export class HealthModule {}

167
packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/app.health.spec.ts Normal file
View File

@ -0,0 +1,167 @@
import { HealthIndicatorService } from '@nestjs/terminus';
import { Test, TestingModule } from '@nestjs/testing';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
describe('AppHealthIndicator', () => {
let service: AppHealthIndicator;
let objectMetadataService: jest.Mocked<ObjectMetadataService>;
let workspaceHealthService: jest.Mocked<WorkspaceHealthService>;
let workspaceMigrationService: jest.Mocked<WorkspaceMigrationService>;
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
beforeEach(async () => {
objectMetadataService = {
findMany: jest.fn(),
} as any;
workspaceHealthService = {
healthCheck: jest.fn(),
} as any;
workspaceMigrationService = {
getPendingMigrations: jest.fn(),
} as any;
healthIndicatorService = {
check: jest.fn().mockReturnValue({
up: jest.fn().mockImplementation((data) => ({
app: { status: 'up', ...data },
})),
down: jest.fn().mockImplementation((data) => ({
app: { status: 'down', ...data },
})),
}),
} as any;
const module: TestingModule = await Test.createTestingModule({
providers: [
AppHealthIndicator,
{
provide: ObjectMetadataService,
useValue: objectMetadataService,
},
{
provide: WorkspaceHealthService,
useValue: workspaceHealthService,
},
{
provide: WorkspaceMigrationService,
useValue: workspaceMigrationService,
},
{
provide: HealthIndicatorService,
useValue: healthIndicatorService,
},
],
}).compile();
service = module.get<AppHealthIndicator>(AppHealthIndicator);
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
it('should return up status when no issues and no pending migrations', async () => {
objectMetadataService.findMany.mockResolvedValue([
{
id: '1',
workspaceId: 'workspace1',
} as any,
{
id: '2',
workspaceId: 'workspace2',
} as any,
]);
workspaceMigrationService.getPendingMigrations.mockResolvedValue([]);
const result = await service.isHealthy();
expect(result.app.status).toBe('up');
expect(result.app.details.overview.totalWorkspacesCount).toBe(2);
expect(result.app.details.overview.criticalWorkspacesCount).toBe(0);
expect(result.app.details.criticalWorkspaces).toBe(null);
expect(result.app.details.system.nodeVersion).toBeDefined();
expect(result.app.details.system.timestamp).toBeDefined();
});
it('should return down status when there are pending migrations', async () => {
objectMetadataService.findMany.mockResolvedValue([
{
id: '1',
workspaceId: 'workspace1',
} as any,
]);
workspaceMigrationService.getPendingMigrations.mockResolvedValue([
{
id: '1',
createdAt: new Date(),
migrations: [],
name: 'migration1',
isCustom: false,
workspaceId: 'workspace1',
} as any,
]);
const result = await service.isHealthy();
expect(result.app.status).toBe('down');
expect(result.app.details.overview.criticalWorkspacesCount).toBe(1);
expect(result.app.details.criticalWorkspaces).toEqual([
{
workspaceId: 'workspace1',
pendingMigrations: 1,
},
]);
});
it('should handle errors gracefully and maintain state history', async () => {
objectMetadataService.findMany.mockRejectedValue(
new Error('Database connection failed'),
);
const result = await service.isHealthy();
expect(result.app.status).toBe('down');
expect(result.app.message).toBe('Database connection failed');
expect(result.app.details.system.nodeVersion).toBeDefined();
expect(result.app.details.system.timestamp).toBeDefined();
expect(result.app.details.stateHistory).toBeDefined();
});
it('should maintain state history across health checks', async () => {
// First check - healthy state
objectMetadataService.findMany.mockResolvedValue([
{
id: '1',
workspaceId: 'workspace1',
} as any,
]);
workspaceMigrationService.getPendingMigrations.mockResolvedValue([]);
await service.isHealthy();
// Second check - error state
objectMetadataService.findMany.mockRejectedValue(
new Error('Database connection failed'),
);
const result = await service.isHealthy();
expect(result.app.details.stateHistory).toBeDefined();
expect(result.app.details.stateHistory.age).toBeDefined();
expect(result.app.details.stateHistory.timestamp).toBeDefined();
expect(result.app.details.stateHistory.details).toBeDefined();
});
});

88
packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts
View File

@ -22,11 +22,8 @@ describe('DatabaseHealthIndicator', () => {
up: jest.fn().mockImplementation((data) => ({
database: { status: 'up', ...data },
})),
down: jest.fn().mockImplementation((error) => ({
database: {
status: 'down',
error,
},
down: jest.fn().mockImplementation((data) => ({
database: { status: 'down', ...data },
})),
}),
} as any;
@ -77,10 +74,20 @@ describe('DatabaseHealthIndicator', () => {
const result = await service.isHealthy();
expect(result.database.status).toBe('up');
expect(result.database.details).toBeDefined();
expect(result.database.details.version).toBeDefined();
expect(result.database.details.connections).toBeDefined();
expect(result.database.details.performance).toBeDefined();
expect(result.database.details.system.version).toBe('PostgreSQL 15.6');
expect(result.database.details.system.timestamp).toBeDefined();
expect(result.database.details.connections).toEqual({
active: 5,
max: 100,
utilizationPercent: 5,
});
expect(result.database.details.performance).toEqual({
cacheHitRatio: '96%',
deadlocks: 0,
slowQueries: 0,
});
expect(result.database.details.databaseSize).toBe('1 GB');
expect(result.database.details.top10Tables).toEqual([{ table_stats: [] }]);
});
it('should return down status when database fails', async () => {
@ -91,9 +98,11 @@ describe('DatabaseHealthIndicator', () => {
const result = await service.isHealthy();
expect(result.database.status).toBe('down');
expect(result.database.error).toBe(
expect(result.database.message).toBe(
HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED,
);
expect(result.database.details.system.timestamp).toBeDefined();
expect(result.database.details.stateHistory).toBeDefined();
});
it('should timeout after specified duration', async () => {
@ -111,6 +120,63 @@ describe('DatabaseHealthIndicator', () => {
const result = await healthCheckPromise;
expect(result.database.status).toBe('down');
expect(result.database.error).toBe(HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT);
expect(result.database.message).toBe(
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
);
expect(result.database.details.stateHistory).toBeDefined();
});
it('should maintain state history across health checks', async () => {
// First check - healthy state
const mockResponses = [
[{ version: 'PostgreSQL 15.6' }],
[{ count: '5' }],
[{ max_connections: '100' }],
[{ uptime: '3600' }],
[{ size: '1 GB' }],
[{ table_stats: [] }],
[{ ratio: '95.5' }],
[{ deadlocks: '0' }],
[{ count: '0' }],
];
mockResponses.forEach((response) => {
dataSource.query.mockResolvedValueOnce(response);
});
const firstResult = await service.isHealthy();
expect(firstResult.database.status).toBe('up');
// Second check - error state
dataSource.query.mockRejectedValueOnce(
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
);
const result = await service.isHealthy();
expect(result.database.details.stateHistory).toMatchObject({
age: expect.any(Number),
timestamp: expect.any(Date),
details: {
system: {
version: 'PostgreSQL 15.6',
timestamp: expect.any(String),
uptime: expect.any(String),
},
connections: {
active: 5,
max: 100,
utilizationPercent: 5,
},
performance: {
cacheHitRatio: '96%',
deadlocks: 0,
slowQueries: 0,
},
databaseSize: '1 GB',
top10Tables: [{ table_stats: [] }],
},
});
});
});

58
packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts
View File

@ -35,7 +35,7 @@ describe('RedisHealthIndicator', () => {
down: jest.fn().mockImplementation((error) => ({
redis: {
status: 'down',
error,
...error,
},
})),
}),
@ -72,35 +72,43 @@ describe('RedisHealthIndicator', () => {
mockRedis.info
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
.mockResolvedValueOnce(
'used_memory_human:1.2G\nused_memory_peak_human:1.5G\nmem_fragmentation_ratio:1.5\n',
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
)
.mockResolvedValueOnce('connected_clients:5\n')
.mockResolvedValueOnce('connected_clients:5\r\n')
.mockResolvedValueOnce(
'total_connections_received:100\nkeyspace_hits:90\nkeyspace_misses:10\n',
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
);
const result = await service.isHealthy();
expect(result.redis.status).toBe('up');
expect(result.redis.details).toBeDefined();
expect(result.redis.details.version).toBe('7.0.0');
expect(result.redis.details.system.version).toBe('7.0.0');
expect(result.redis.details.system.timestamp).toBeDefined();
expect(result.redis.details.memory).toEqual({
used: '1.2G',
peak: '1.5G',
fragmentation: 1.5,
});
});
it('should return down status when redis fails', async () => {
mockRedis.ping.mockRejectedValueOnce(
mockRedis.info.mockRejectedValueOnce(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
const result = await service.isHealthy();
expect(result.redis.status).toBe('down');
expect(result.redis.error).toBe(
expect(result.redis.message).toBe(
HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
);
expect(result.redis.details.system.timestamp).toBeDefined();
expect(result.redis.details.stateHistory).toBeDefined();
});
it('should timeout after specified duration', async () => {
mockRedis.ping.mockImplementationOnce(
mockRedis.info.mockImplementationOnce(
() =>
new Promise((resolve) =>
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
@ -110,24 +118,38 @@ describe('RedisHealthIndicator', () => {
const healthCheckPromise = service.isHealthy();
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
const result = await healthCheckPromise;
expect(result.redis.status).toBe('down');
expect(result.redis.error).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT);
expect(result.redis.message).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT);
expect(result.redis.details.system.timestamp).toBeDefined();
expect(result.redis.details.stateHistory).toBeDefined();
});
it('should handle partial failures in health details collection', async () => {
it('should maintain state history across health checks', async () => {
// First check - healthy state
mockRedis.info
.mockResolvedValueOnce('redis_version:7.0.0') // info
.mockResolvedValueOnce('used_memory_human:1.2G') // memory
.mockResolvedValueOnce('connected_clients:5') // clients
.mockResolvedValueOnce('total_connections_received:100'); // stats
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
.mockResolvedValueOnce(
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
)
.mockResolvedValueOnce('connected_clients:5\r\n')
.mockResolvedValueOnce(
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
);
await service.isHealthy();
// Second check - error state
mockRedis.info.mockRejectedValueOnce(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
const result = await service.isHealthy();
expect(result.redis.status).toBe('up');
expect(result.redis.details).toBeDefined();
expect(result.redis.details.version).toBe('7.0.0');
expect(result.redis.details.stateHistory).toBeDefined();
expect(result.redis.details.stateHistory.age).toBeDefined();
expect(result.redis.details.stateHistory.timestamp).toBeDefined();
expect(result.redis.details.stateHistory.details).toBeDefined();
});
});

96
packages/twenty-server/src/engine/core-modules/health/indicators/app.health.ts Normal file
View File

@ -0,0 +1,96 @@
import { Injectable } from '@nestjs/common';
import {
HealthIndicatorResult,
HealthIndicatorService,
} from '@nestjs/terminus';
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
@Injectable()
export class AppHealthIndicator {
private stateManager = new HealthStateManager();
constructor(
private readonly healthIndicatorService: HealthIndicatorService,
private readonly workspaceHealthService: WorkspaceHealthService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
) {}
async isHealthy(): Promise<HealthIndicatorResult> {
const indicator = this.healthIndicatorService.check('app');
try {
const workspaces = await this.objectMetadataService.findMany();
const workspaceIds = [...new Set(workspaces.map((w) => w.workspaceId))];
const workspaceStats = await Promise.all(
workspaceIds.map(async (workspaceId) => {
const pendingMigrations =
await this.workspaceMigrationService.getPendingMigrations(
workspaceId,
);
return {
workspaceId,
pendingMigrations: pendingMigrations.length,
isCritical: pendingMigrations.length > 0,
};
}),
);
const details = {
system: {
nodeVersion: process.version,
timestamp: new Date().toISOString(),
},
overview: {
totalWorkspacesCount: workspaceIds.length,
criticalWorkspacesCount: workspaceStats.filter(
(stat) => stat.isCritical,
).length,
},
criticalWorkspaces:
workspaceStats.filter((stat) => stat.isCritical).length > 0
? workspaceStats
.filter((stat) => stat.isCritical)
.map((stat) => ({
workspaceId: stat.workspaceId,
pendingMigrations: stat.pendingMigrations,
}))
: null,
};
const isHealthy = workspaceStats.every((stat) => !stat.isCritical);
if (isHealthy) {
this.stateManager.updateState(details);
return indicator.up({ details });
}
this.stateManager.updateState(details);
return indicator.down({
message: `Found ${details.criticalWorkspaces?.length} workspaces with pending migrations`,
details,
});
} catch (error) {
const stateWithAge = this.stateManager.getStateWithAge();
return indicator.down({
message: error.message,
details: {
system: {
nodeVersion: process.version,
timestamp: new Date().toISOString(),
},
stateHistory: stateWithAge,
},
});
}
}
}

60
packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts
View File

@ -9,9 +9,12 @@ import { DataSource } from 'typeorm';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
@Injectable()
export class DatabaseHealthIndicator {
private stateManager = new HealthStateManager();
constructor(
@InjectDataSource('core')
private readonly dataSource: DataSource,
@ -69,35 +72,50 @@ export class DatabaseHealthIndicator {
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
);
return indicator.up({
details: {
const details = {
system: {
timestamp: new Date().toISOString(),
version: versionResult.version,
connections: {
active: parseInt(activeConnections.count),
max: parseInt(maxConnections.max_connections),
utilizationPercent: Math.round(
(parseInt(activeConnections.count) /
parseInt(maxConnections.max_connections)) *
100,
),
},
uptime: Math.round(uptime.uptime / 3600) + ' hours',
databaseSize: databaseSize.size,
performance: {
cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%',
deadlocks: parseInt(deadlocks.deadlocks),
slowQueries: parseInt(slowQueries.count),
},
top10Tables: tableStats,
},
});
connections: {
active: parseInt(activeConnections.count),
max: parseInt(maxConnections.max_connections),
utilizationPercent: Math.round(
(parseInt(activeConnections.count) /
parseInt(maxConnections.max_connections)) *
100,
),
},
databaseSize: databaseSize.size,
performance: {
cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%',
deadlocks: parseInt(deadlocks.deadlocks),
slowQueries: parseInt(slowQueries.count),
},
top10Tables: tableStats,
};
this.stateManager.updateState(details);
return indicator.up({ details });
} catch (error) {
const errorMessage =
const message =
error.message === HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
? HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
: HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED;
return indicator.down(errorMessage);
const stateWithAge = this.stateManager.getStateWithAge();
return indicator.down({
message,
details: {
system: {
timestamp: new Date().toISOString(),
},
stateHistory: stateWithAge,
},
});
}
}
}

82
packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts
View File

@ -6,10 +6,13 @@ import {
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
@Injectable()
export class RedisHealthIndicator {
private stateManager = new HealthStateManager();
constructor(
private readonly redisClient: RedisClientService,
private readonly healthIndicatorService: HealthIndicatorService,
@ -48,47 +51,62 @@ export class RedisHealthIndicator {
const clientsData = parseInfo(clients);
const statsData = parseInfo(stats);
return indicator.up({
details: {
const details = {
system: {
timestamp: new Date().toISOString(),
version: infoData.redis_version,
uptime:
Math.round(parseInt(infoData.uptime_in_seconds) / 3600) + ' hours',
memory: {
used: memoryData.used_memory_human,
peak: memoryData.used_memory_peak_human,
fragmentation: parseFloat(memoryData.mem_fragmentation_ratio),
},
connections: {
current: parseInt(clientsData.connected_clients),
total: parseInt(statsData.total_connections_received),
rejected: parseInt(statsData.rejected_connections),
},
performance: {
opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec),
hitRate: statsData.keyspace_hits
? Math.round(
(parseInt(statsData.keyspace_hits) /
(parseInt(statsData.keyspace_hits) +
parseInt(statsData.keyspace_misses))) *
100,
) + '%'
: '0%',
evictedKeys: parseInt(statsData.evicted_keys),
expiredKeys: parseInt(statsData.expired_keys),
},
replication: {
role: infoData.role,
connectedSlaves: parseInt(infoData.connected_slaves || '0'),
},
},
});
memory: {
used: memoryData.used_memory_human,
peak: memoryData.used_memory_peak_human,
fragmentation: parseFloat(memoryData.mem_fragmentation_ratio),
},
connections: {
current: parseInt(clientsData.connected_clients),
total: parseInt(statsData.total_connections_received),
rejected: parseInt(statsData.rejected_connections),
},
performance: {
opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec),
hitRate: statsData.keyspace_hits
? Math.round(
(parseInt(statsData.keyspace_hits) /
(parseInt(statsData.keyspace_hits) +
parseInt(statsData.keyspace_misses))) *
100,
) + '%'
: '0%',
evictedKeys: parseInt(statsData.evicted_keys),
expiredKeys: parseInt(statsData.expired_keys),
},
replication: {
role: infoData.role,
connectedSlaves: parseInt(infoData.connected_slaves || '0'),
},
};
this.stateManager.updateState(details);
return indicator.up({ details });
} catch (error) {
const errorMessage =
const message =
error.message === HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
? HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED;
return indicator.down(errorMessage);
const stateWithAge = this.stateManager.getStateWithAge();
return indicator.down({
message,
details: {
system: {
timestamp: new Date().toISOString(),
},
stateHistory: stateWithAge,
},
});
}
}
}

22
packages/twenty-server/src/engine/core-modules/health/utils/health-state-manager.util.ts Normal file
View File

@ -0,0 +1,22 @@
export class HealthStateManager {
private lastKnownState: {
timestamp: Date;
details: Record<string, any>;
} | null = null;
updateState(details: Record<string, any>) {
this.lastKnownState = {
timestamp: new Date(),
details,
};
}
getStateWithAge() {
return this.lastKnownState
? {
...this.lastKnownState,
age: Date.now() - this.lastKnownState.timestamp.getTime(),
}
: 'No previous state available';
}
}