From d6655a2c3b4f46a51546c4ba807ac82046dfc720 Mon Sep 17 00:00:00 2001 From: nitin <142569587+ehconitin@users.noreply.github.com> Date: Tue, 18 Feb 2025 20:22:19 +0530 Subject: [PATCH] Health monitor status for admin panel (#10186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Health Monitoring for Self-Hosted Instances This PR implements basic health monitoring for self-hosted instances in the admin panel. ## Service Status Checks We're adding real-time health checks for: - Redis Connection - Database Connection - Worker Status - Message Sync Status ## Existing Functionality We already have message sync and captcha counters that store aggregated metrics in cache within a configurable time window (default: 5 minutes). ## New Endpoints 1. `/healthz` - Basic server health check for Kubernetes pod monitoring 2. `/healthz/{serviceName}` - Individual service health checks (returns 200 if healthy) 3. `/metricsz/{metricName}` - Time-windowed metrics (message sync, captcha) 4. GraphQL resolver in admin panel for UI consumption All endpoints use the same underlying service, with different presentation layers for infrastructure and UI needs. --------- Co-authored-by: Félix Malfait --- package.json | 2 +- .../twenty-front/src/generated/graphql.tsx | 174 ++++++++++++++++ .../modules/app/components/SettingsRoutes.tsx | 12 ++ .../components/SettingsAdminContent.tsx | 7 +- .../components/SettingsAdminGeneral.tsx | 9 + ...ngsAdminHealthMessageSyncCountersTable.tsx | 44 ++++ .../components/SettingsAdminHealthStatus.tsx | 66 ++++++ ...ettingsAdminHealthStatusRightContainer.tsx | 27 +++ .../SettingsAdminQueueExpandableContainer.tsx | 103 ++++++++++ .../SettingsAdminQueueHealthButtons.tsx | 52 +++++ .../components/SettingsAdminTabContent.tsx | 3 + .../SettingsHealthStatusListCard.tsx | 43 ++++ .../constants/SettingsAdminTabs.ts | 1 + .../queries/getIndicatorHealthStatus.ts | 25 +++ .../graphql/queries/getSystemHealthStatus.ts | 36 ++++ .../hooks/useGetUptoDateHealthStatus.ts | 12 ++ .../admin-panel/types/AdminHealthService.ts | 12 ++ .../src/modules/types/SettingsPath.ts | 2 + .../src/pages/settings/SettingsWorkspace.tsx | 12 +- .../SettingsAdminIndicatorHealthStatus.tsx | 137 +++++++++++++ .../admin-panel-health.service.spec.ts | 193 ++++++++++++++++++ ...el.spec.ts => admin-panel.service.spec.ts} | 0 .../admin-panel/admin-panel-health.service.ts | 108 ++++++++++ .../admin-panel/admin-panel.module.ts | 11 +- .../admin-panel/admin-panel.resolver.ts | 26 ++- .../admin-panel-health-service-data.dto.ts | 16 ++ ...min-panel-indicator-health-status.input.ts | 17 ++ .../admin-panel-worker-queue-health.dto.ts | 10 + .../admin-panel/dtos/system-health.dto.ts | 18 ++ .../admin-panel-health-service-status.enum.ts | 10 + .../health-error-messages.constants.ts | 12 ++ .../health-indicators-timeout.conts.ts | 1 + .../__tests__/health.controller.spec.ts | 41 ++++ .../__tests__/metrics.controller.spec.ts | 29 +++ .../health/controllers/health.controller.ts | 39 ++++ .../metrics.controller.ts} | 16 +- .../health/enums/health-service-name.enum.ts | 6 + .../health/health-cache.service.ts | 2 +- .../health/health.controller.spec.ts | 36 ---- .../core-modules/health/health.module.ts | 31 ++- .../__tests__/database.health.spec.ts | 116 +++++++++++ .../__tests__/message-sync.health.spec.ts | 137 +++++++++++++ .../indicators/__tests__/redis.health.spec.ts | 133 ++++++++++++ .../__tests__/worker.health.spec.ts | 136 ++++++++++++ .../health/indicators/database.health.ts | 103 ++++++++++ .../health/indicators/message-sync.health.ts | 63 ++++++ .../health/indicators/redis.health.ts | 94 +++++++++ .../health/indicators/worker.health.ts | 103 ++++++++++ .../health/types/health-metrics.types.ts | 5 - .../types/message-sync-metrics.types.ts | 19 ++ .../health/types/worker-queue-health.type.ts | 15 ++ .../health/types/worker-queue-metrics.type.ts | 22 ++ .../health/utils/health-check-timeout.util.ts | 16 ++ yarn.lock | 39 ++-- 54 files changed, 2307 insertions(+), 95 deletions(-) create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable.tsx create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatus.tsx create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatusRightContainer.tsx create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueExpandableContainer.tsx create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueHealthButtons.tsx create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/components/SettingsHealthStatusListCard.tsx create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getIndicatorHealthStatus.ts create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getSystemHealthStatus.ts create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/hooks/useGetUptoDateHealthStatus.ts create mode 100644 packages/twenty-front/src/modules/settings/admin-panel/types/AdminHealthService.ts create mode 100644 packages/twenty-front/src/pages/settings/admin-panel/SettingsAdminIndicatorHealthStatus.tsx create mode 100644 packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts rename packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/{admin-panel.spec.ts => admin-panel.service.spec.ts} (100%) create mode 100644 packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts create mode 100644 packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts create mode 100644 packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input.ts create mode 100644 packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto.ts create mode 100644 packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts create mode 100644 packages/twenty-server/src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/constants/health-indicators-timeout.conts.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts rename packages/twenty-server/src/engine/core-modules/health/{health.controller.ts => controllers/metrics.controller.ts} (59%) create mode 100644 packages/twenty-server/src/engine/core-modules/health/enums/health-service-name.enum.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/health/health.controller.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/message-sync.health.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/worker.health.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/message-sync.health.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/health/types/health-metrics.types.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/types/message-sync-metrics.types.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/types/worker-queue-metrics.type.ts create mode 100644 packages/twenty-server/src/engine/core-modules/health/utils/health-check-timeout.util.ts diff --git a/package.json b/package.json index 6514a0be3..79261e146 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "@nestjs/passport": "^9.0.3", "@nestjs/platform-express": "^9.0.0", "@nestjs/serve-static": "^4.0.1", - "@nestjs/terminus": "^9.2.2", + "@nestjs/terminus": "^11.0.0", "@nestjs/typeorm": "^10.0.0", "@nx/eslint-plugin": "^17.2.8", "@octokit/graphql": "^7.0.2", diff --git a/packages/twenty-front/src/generated/graphql.tsx b/packages/twenty-front/src/generated/graphql.tsx index cb2c72dae..a03fb87ab 100644 --- a/packages/twenty-front/src/generated/graphql.tsx +++ b/packages/twenty-front/src/generated/graphql.tsx @@ -25,6 +25,33 @@ export type ActivateWorkspaceInput = { displayName?: InputMaybe; }; +export type AdminPanelHealthServiceData = { + __typename?: 'AdminPanelHealthServiceData'; + details?: Maybe; + queues?: Maybe>; + status: AdminPanelHealthServiceStatus; +}; + +export enum AdminPanelHealthServiceStatus { + OPERATIONAL = 'OPERATIONAL', + OUTAGE = 'OUTAGE' +} + +export enum AdminPanelIndicatorHealthStatusInputEnum { + DATABASE = 'DATABASE', + MESSAGE_SYNC = 'MESSAGE_SYNC', + REDIS = 'REDIS', + WORKER = 'WORKER' +} + +export type AdminPanelWorkerQueueHealth = { + __typename?: 'AdminPanelWorkerQueueHealth'; + metrics: WorkerQueueMetrics; + name: Scalars['String']; + status: AdminPanelHealthServiceStatus; + workers: Scalars['Float']; +}; + export type Analytics = { __typename?: 'Analytics'; /** Boolean that confirms query was dispatched */ @@ -1232,11 +1259,13 @@ export type Query = { findWorkspaceInvitations: Array; getAvailablePackages: Scalars['JSON']; getEnvironmentVariablesGrouped: EnvironmentVariablesOutput; + getIndicatorHealthStatus: AdminPanelHealthServiceData; getPostgresCredentials?: Maybe; getProductPrices: BillingProductPricesOutput; getPublicWorkspaceDataByDomain: PublicWorkspaceDataOutput; getRoles: Array; getServerlessFunctionSourceCode?: Maybe; + getSystemHealthStatus: SystemHealth; getTimelineCalendarEventsFromCompanyId: TimelineCalendarEventsWithTotal; getTimelineCalendarEventsFromPersonId: TimelineCalendarEventsWithTotal; getTimelineThreadsFromCompanyId: TimelineThreadsWithTotal; @@ -1287,6 +1316,11 @@ export type QueryGetAvailablePackagesArgs = { }; +export type QueryGetIndicatorHealthStatusArgs = { + indicatorName: AdminPanelIndicatorHealthStatusInputEnum; +}; + + export type QueryGetProductPricesArgs = { product: Scalars['String']; }; @@ -1605,6 +1639,14 @@ export type Support = { supportFrontChatId?: Maybe; }; +export type SystemHealth = { + __typename?: 'SystemHealth'; + database: AdminPanelHealthServiceData; + messageSync: AdminPanelHealthServiceData; + redis: AdminPanelHealthServiceData; + worker: AdminPanelHealthServiceData; +}; + export type TimelineCalendarEvent = { __typename?: 'TimelineCalendarEvent'; conferenceLink: LinksMetadata; @@ -1854,6 +1896,16 @@ export type ValidatePasswordResetToken = { id: Scalars['String']; }; +export type WorkerQueueMetrics = { + __typename?: 'WorkerQueueMetrics'; + active: Scalars['Float']; + completed: Scalars['Float']; + delayed: Scalars['Float']; + failed: Scalars['Float']; + prioritized: Scalars['Float']; + waiting: Scalars['Float']; +}; + export type WorkflowAction = { __typename?: 'WorkflowAction'; id: Scalars['UUID']; @@ -2250,6 +2302,18 @@ export type GetEnvironmentVariablesGroupedQueryVariables = Exact<{ [key: string] export type GetEnvironmentVariablesGroupedQuery = { __typename?: 'Query', getEnvironmentVariablesGrouped: { __typename?: 'EnvironmentVariablesOutput', groups: Array<{ __typename?: 'EnvironmentVariablesGroupData', name: EnvironmentVariablesGroup, description: string, isHiddenOnLoad: boolean, variables: Array<{ __typename?: 'EnvironmentVariable', name: string, description: string, value: string, sensitive: boolean }> }> } }; +export type GetIndicatorHealthStatusQueryVariables = Exact<{ + indicatorName: AdminPanelIndicatorHealthStatusInputEnum; +}>; + + +export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', name: string, status: AdminPanelHealthServiceStatus, workers: number, metrics: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, prioritized: number } }> | null } }; + +export type GetSystemHealthStatusQueryVariables = Exact<{ [key: string]: never; }>; + + +export type GetSystemHealthStatusQuery = { __typename?: 'Query', getSystemHealthStatus: { __typename?: 'SystemHealth', database: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null }, redis: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null }, worker: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', name: string, workers: number, status: AdminPanelHealthServiceStatus, metrics: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, prioritized: number } }> | null }, messageSync: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null } } }; + export type UpdateLabPublicFeatureFlagMutationVariables = Exact<{ input: UpdateLabPublicFeatureFlagInput; }>; @@ -3960,6 +4024,116 @@ export function useGetEnvironmentVariablesGroupedLazyQuery(baseOptions?: Apollo. export type GetEnvironmentVariablesGroupedQueryHookResult = ReturnType; export type GetEnvironmentVariablesGroupedLazyQueryHookResult = ReturnType; export type GetEnvironmentVariablesGroupedQueryResult = Apollo.QueryResult; +export const GetIndicatorHealthStatusDocument = gql` + query GetIndicatorHealthStatus($indicatorName: AdminPanelIndicatorHealthStatusInputEnum!) { + getIndicatorHealthStatus(indicatorName: $indicatorName) { + status + details + queues { + name + status + workers + metrics { + failed + completed + waiting + active + delayed + prioritized + } + } + } +} + `; + +/** + * __useGetIndicatorHealthStatusQuery__ + * + * To run a query within a React component, call `useGetIndicatorHealthStatusQuery` and pass it any options that fit your needs. + * When your component renders, `useGetIndicatorHealthStatusQuery` returns an object from Apollo Client that contains loading, error, and data properties + * you can use to render your UI. + * + * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options; + * + * @example + * const { data, loading, error } = useGetIndicatorHealthStatusQuery({ + * variables: { + * indicatorName: // value for 'indicatorName' + * }, + * }); + */ +export function useGetIndicatorHealthStatusQuery(baseOptions: Apollo.QueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useQuery(GetIndicatorHealthStatusDocument, options); + } +export function useGetIndicatorHealthStatusLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useLazyQuery(GetIndicatorHealthStatusDocument, options); + } +export type GetIndicatorHealthStatusQueryHookResult = ReturnType; +export type GetIndicatorHealthStatusLazyQueryHookResult = ReturnType; +export type GetIndicatorHealthStatusQueryResult = Apollo.QueryResult; +export const GetSystemHealthStatusDocument = gql` + query GetSystemHealthStatus { + getSystemHealthStatus { + database { + status + details + } + redis { + status + details + } + worker { + status + queues { + name + workers + status + metrics { + failed + completed + waiting + active + delayed + prioritized + } + } + } + messageSync { + status + details + } + } +} + `; + +/** + * __useGetSystemHealthStatusQuery__ + * + * To run a query within a React component, call `useGetSystemHealthStatusQuery` and pass it any options that fit your needs. + * When your component renders, `useGetSystemHealthStatusQuery` returns an object from Apollo Client that contains loading, error, and data properties + * you can use to render your UI. + * + * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options; + * + * @example + * const { data, loading, error } = useGetSystemHealthStatusQuery({ + * variables: { + * }, + * }); + */ +export function useGetSystemHealthStatusQuery(baseOptions?: Apollo.QueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useQuery(GetSystemHealthStatusDocument, options); + } +export function useGetSystemHealthStatusLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useLazyQuery(GetSystemHealthStatusDocument, options); + } +export type GetSystemHealthStatusQueryHookResult = ReturnType; +export type GetSystemHealthStatusLazyQueryHookResult = ReturnType; +export type GetSystemHealthStatusQueryResult = Apollo.QueryResult; export const UpdateLabPublicFeatureFlagDocument = gql` mutation UpdateLabPublicFeatureFlag($input: UpdateLabPublicFeatureFlagInput!) { updateLabPublicFeatureFlag(input: $input) { diff --git a/packages/twenty-front/src/modules/app/components/SettingsRoutes.tsx b/packages/twenty-front/src/modules/app/components/SettingsRoutes.tsx index 04b7e7a73..d3e6448e1 100644 --- a/packages/twenty-front/src/modules/app/components/SettingsRoutes.tsx +++ b/packages/twenty-front/src/modules/app/components/SettingsRoutes.tsx @@ -248,6 +248,14 @@ const SettingsAdminContent = lazy(() => ), ); +const SettingsAdminIndicatorHealthStatus = lazy(() => + import( + '~/pages/settings/admin-panel/SettingsAdminIndicatorHealthStatus' + ).then((module) => ({ + default: module.SettingsAdminIndicatorHealthStatus, + })), +); + const SettingsLab = lazy(() => import('~/pages/settings/lab/SettingsLab').then((module) => ({ default: module.SettingsLab, @@ -407,6 +415,10 @@ export const SettingsRoutes = ({ path={SettingsPath.FeatureFlags} element={} /> + } + /> )} } /> diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminContent.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminContent.tsx index ff4118fcd..547939713 100644 --- a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminContent.tsx +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminContent.tsx @@ -3,7 +3,7 @@ import { SETTINGS_ADMIN_TABS } from '@/settings/admin-panel/constants/SettingsAd import { SETTINGS_ADMIN_TABS_ID } from '@/settings/admin-panel/constants/SettingsAdminTabsId'; import { TabList } from '@/ui/layout/tab/components/TabList'; import styled from '@emotion/styled'; -import { IconSettings2, IconVariable } from 'twenty-ui'; +import { IconHeart, IconSettings2, IconVariable } from 'twenty-ui'; const StyledTabListContainer = styled.div` align-items: center; @@ -25,6 +25,11 @@ export const SettingsAdminContent = () => { title: 'Env Variables', Icon: IconVariable, }, + { + id: SETTINGS_ADMIN_TABS.HEALTH_STATUS, + title: 'Health Status', + Icon: IconHeart, + }, ]; return ( diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminGeneral.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminGeneral.tsx index 0632b0966..7ba095ff9 100644 --- a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminGeneral.tsx +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminGeneral.tsx @@ -15,6 +15,7 @@ import { useRecoilState, useRecoilValue } from 'recoil'; import { getImageAbsoluteURI, isDefined } from 'twenty-shared'; import { Button, + GithubVersionLink, H1Title, H1TitleFontColor, H2Title, @@ -24,6 +25,8 @@ import { import { REACT_APP_SERVER_BASE_URL } from '~/config'; import { useUserLookupAdminPanelMutation } from '~/generated/graphql'; +import packageJson from '../../../../../package.json'; + const StyledLinkContainer = styled.div` margin-right: ${({ theme }) => theme.spacing(2)}; width: 100%; @@ -120,6 +123,11 @@ export const SettingsAdminGeneral = () => { return ( <> +
+ + +
+
{ behaveAsLinks={false} /> + diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable.tsx new file mode 100644 index 000000000..827120658 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable.tsx @@ -0,0 +1,44 @@ +import { Table } from '@/ui/layout/table/components/Table'; +import { TableCell } from '@/ui/layout/table/components/TableCell'; +import { TableHeader } from '@/ui/layout/table/components/TableHeader'; +import { TableRow } from '@/ui/layout/table/components/TableRow'; + +export const SettingsAdminHealthMessageSyncCountersTable = ({ + details, +}: { + details: string | null | undefined; +}) => { + const parsedDetails = details ? JSON.parse(details) : null; + if (!parsedDetails) { + return null; + } + + return ( + + + Status + Count + + + Message Not Synced + {parsedDetails.counters.NOT_SYNCED} + + + Message Sync Ongoing + {parsedDetails.counters.ONGOING} + + + Total Jobs + {parsedDetails.totalJobs} + + + Failed Jobs + {parsedDetails.failedJobs} + + + Failure Rate + {parsedDetails.failureRate}% + +
+ ); +}; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatus.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatus.tsx new file mode 100644 index 000000000..391326cf2 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatus.tsx @@ -0,0 +1,66 @@ +import { SettingsAdminHealthMessageSyncCountersTable } from '@/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable'; +import { SettingsHealthStatusListCard } from '@/settings/admin-panel/components/SettingsHealthStatusListCard'; +import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService'; +import styled from '@emotion/styled'; +import { H2Title, Section } from 'twenty-ui'; +import { + AdminPanelHealthServiceStatus, + useGetSystemHealthStatusQuery, +} from '~/generated/graphql'; + +const StyledErrorMessage = styled.div` + color: ${({ theme }) => theme.color.red}; + margin-top: ${({ theme }) => theme.spacing(2)}; +`; + +export const SettingsAdminHealthStatus = () => { + const { data, loading } = useGetSystemHealthStatusQuery({ + fetchPolicy: 'network-only', + }); + + const services = [ + { + id: 'DATABASE', + name: 'Database Status', + ...data?.getSystemHealthStatus.database, + }, + { id: 'REDIS', name: 'Redis Status', ...data?.getSystemHealthStatus.redis }, + { + id: 'WORKER', + name: 'Worker Status', + status: data?.getSystemHealthStatus.worker.status, + queues: data?.getSystemHealthStatus.worker.queues, + }, + ].filter((service): service is AdminHealthService => !!service.status); + + const isMessageSyncCounterDown = + !data?.getSystemHealthStatus.messageSync.status || + data?.getSystemHealthStatus.messageSync.status === + AdminPanelHealthServiceStatus.OUTAGE; + + return ( + <> +
+ + +
+ +
+ + {isMessageSyncCounterDown ? ( + + {data?.getSystemHealthStatus.messageSync.details || + 'Message sync status is unavailable'} + + ) : ( + + )} +
+ + ); +}; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatusRightContainer.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatusRightContainer.tsx new file mode 100644 index 000000000..9ce2372e8 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatusRightContainer.tsx @@ -0,0 +1,27 @@ +import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService'; +import styled from '@emotion/styled'; +import { Status } from 'twenty-ui'; +import { AdminPanelHealthServiceStatus } from '~/generated/graphql'; + +const StyledRowRightContainer = styled.div` + align-items: center; + display: flex; + gap: ${({ theme }) => theme.spacing(1)}; +`; + +export const SettingsAdminHealthStatusRightContainer = ({ + service, +}: { + service: AdminHealthService; +}) => { + return ( + + {service.status === AdminPanelHealthServiceStatus.OPERATIONAL && ( + + )} + {service.status === AdminPanelHealthServiceStatus.OUTAGE && ( + + )} + + ); +}; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueExpandableContainer.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueExpandableContainer.tsx new file mode 100644 index 000000000..01593e622 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueExpandableContainer.tsx @@ -0,0 +1,103 @@ +import { SettingsListCard } from '@/settings/components/SettingsListCard'; +import { Table } from '@/ui/layout/table/components/Table'; +import { TableCell } from '@/ui/layout/table/components/TableCell'; +import { TableRow } from '@/ui/layout/table/components/TableRow'; +import styled from '@emotion/styled'; +import { AnimatedExpandableContainer, Status } from 'twenty-ui'; +import { + AdminPanelHealthServiceStatus, + AdminPanelWorkerQueueHealth, +} from '~/generated/graphql'; + +const StyledExpandedContent = styled.div` + border: 1px solid ${({ theme }) => theme.border.color.medium}; + border-radius: ${({ theme }) => theme.border.radius.sm}; + padding-top: ${({ theme }) => theme.spacing(1)}; + padding-bottom: ${({ theme }) => theme.spacing(3)}; + padding-left: ${({ theme }) => theme.spacing(3)}; + padding-right: ${({ theme }) => theme.spacing(3)}; +`; + +const StyledContainer = styled.div` + margin-bottom: ${({ theme }) => theme.spacing(3)}; + margin-top: ${({ theme }) => theme.spacing(5)}; +`; + +const StyledTableRow = styled(TableRow)` + height: ${({ theme }) => theme.spacing(6)}; +`; + +const StyledQueueMetricsTitle = styled.div` + color: ${({ theme }) => theme.font.color.primary}; + font-size: ${({ theme }) => theme.font.size.sm}; + font-weight: ${({ theme }) => theme.font.weight.medium}; + margin-bottom: ${({ theme }) => theme.spacing(3)}; + padding-left: ${({ theme }) => theme.spacing(3)}; +`; + +export const SettingsAdminQueueExpandableContainer = ({ + queues, + selectedQueue, +}: { + queues: AdminPanelWorkerQueueHealth[]; + selectedQueue: string | null; +}) => { + const selectedQueueData = queues.find( + (queue) => queue.name === selectedQueue, + ); + + return ( + + {selectedQueueData && ( + <> + + item.name} + isLoading={false} + RowRightComponent={({ + item, + }: { + item: AdminPanelWorkerQueueHealth; + }) => ( + + )} + /> + + Metrics: + + + + Workers + {selectedQueueData.workers} + + {Object.entries(selectedQueueData.metrics) + .filter(([key]) => key !== '__typename') + .map(([key, value]) => ( + + + {key.charAt(0).toUpperCase() + key.slice(1)} + + {value} + + ))} +
+ + + )} + + ); +}; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueHealthButtons.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueHealthButtons.tsx new file mode 100644 index 000000000..8cd51336e --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueHealthButtons.tsx @@ -0,0 +1,52 @@ +import styled from '@emotion/styled'; +import { Button } from 'twenty-ui'; +import { + AdminPanelHealthServiceStatus, + AdminPanelWorkerQueueHealth, +} from '~/generated/graphql'; + +const StyledQueueButtonsRow = styled.div` + display: flex; + flex-wrap: wrap; + gap: ${({ theme }) => theme.spacing(2)}; + margin-top: ${({ theme }) => theme.spacing(6)}; +`; + +const StyledQueueHealthButton = styled(Button)<{ + isSelected?: boolean; + status: AdminPanelHealthServiceStatus; +}>` + ${({ isSelected, theme, status }) => + isSelected && + ` + background-color: ${ + status === AdminPanelHealthServiceStatus.OPERATIONAL + ? theme.tag.background.green + : theme.tag.background.red + }; + `} +`; +export const SettingsAdminQueueHealthButtons = ({ + queues, + selectedQueue, + toggleQueueVisibility, +}: { + queues: AdminPanelWorkerQueueHealth[]; + selectedQueue: string | null; + toggleQueueVisibility: (queueName: string) => void; +}) => { + return ( + + {queues.map((queue) => ( + toggleQueueVisibility(queue.name)} + title={queue.name} + variant="secondary" + isSelected={selectedQueue === queue.name} + status={queue.status} + /> + ))} + + ); +}; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminTabContent.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminTabContent.tsx index f032b3d53..b67e6476c 100644 --- a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminTabContent.tsx +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminTabContent.tsx @@ -1,5 +1,6 @@ import { SettingsAdminEnvVariables } from '@/settings/admin-panel/components/SettingsAdminEnvVariables'; import { SettingsAdminGeneral } from '@/settings/admin-panel/components/SettingsAdminGeneral'; +import { SettingsAdminHealthStatus } from '@/settings/admin-panel/components/SettingsAdminHealthStatus'; import { SETTINGS_ADMIN_TABS } from '@/settings/admin-panel/constants/SettingsAdminTabs'; import { SETTINGS_ADMIN_TABS_ID } from '@/settings/admin-panel/constants/SettingsAdminTabsId'; import { useTabList } from '@/ui/layout/tab/hooks/useTabList'; @@ -12,6 +13,8 @@ export const SettingsAdminTabContent = () => { return ; case SETTINGS_ADMIN_TABS.ENV_VARIABLES: return ; + case SETTINGS_ADMIN_TABS.HEALTH_STATUS: + return ; default: return null; } diff --git a/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsHealthStatusListCard.tsx b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsHealthStatusListCard.tsx new file mode 100644 index 000000000..939805d59 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/components/SettingsHealthStatusListCard.tsx @@ -0,0 +1,43 @@ +import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService'; +import styled from '@emotion/styled'; + +import { SettingsPath } from '@/types/SettingsPath'; +import { Link } from 'react-router-dom'; +import { getSettingsPath } from '~/utils/navigation/getSettingsPath'; +import { SettingsListCard } from '../../components/SettingsListCard'; +import { SettingsAdminHealthStatusRightContainer } from './SettingsAdminHealthStatusRightContainer'; + +const StyledLink = styled(Link)` + text-decoration: none; +`; + +export const SettingsHealthStatusListCard = ({ + services, + loading, +}: { + services: Array; + loading?: boolean; +}) => { + return ( + <> + {services.map((service) => ( + <> + + service.name} + isLoading={loading} + RowRightComponent={({ item: service }) => ( + + )} + /> + + + ))} + + ); +}; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/constants/SettingsAdminTabs.ts b/packages/twenty-front/src/modules/settings/admin-panel/constants/SettingsAdminTabs.ts index fe9d1be5f..f3150e3b4 100644 --- a/packages/twenty-front/src/modules/settings/admin-panel/constants/SettingsAdminTabs.ts +++ b/packages/twenty-front/src/modules/settings/admin-panel/constants/SettingsAdminTabs.ts @@ -1,4 +1,5 @@ export const SETTINGS_ADMIN_TABS = { GENERAL: 'general', ENV_VARIABLES: 'env-variables', + HEALTH_STATUS: 'health-status', }; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getIndicatorHealthStatus.ts b/packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getIndicatorHealthStatus.ts new file mode 100644 index 000000000..dff92ce99 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getIndicatorHealthStatus.ts @@ -0,0 +1,25 @@ +import { gql } from '@apollo/client'; + +export const GET_INDICATOR_HEALTH_STATUS = gql` + query GetIndicatorHealthStatus( + $indicatorName: AdminPanelIndicatorHealthStatusInputEnum! + ) { + getIndicatorHealthStatus(indicatorName: $indicatorName) { + status + details + queues { + name + status + workers + metrics { + failed + completed + waiting + active + delayed + prioritized + } + } + } + } +`; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getSystemHealthStatus.ts b/packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getSystemHealthStatus.ts new file mode 100644 index 000000000..7fc80f35f --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getSystemHealthStatus.ts @@ -0,0 +1,36 @@ +import { gql } from '@apollo/client'; + +export const GET_SYSTEM_HEALTH_STATUS = gql` + query GetSystemHealthStatus { + getSystemHealthStatus { + database { + status + details + } + redis { + status + details + } + worker { + status + queues { + name + workers + status + metrics { + failed + completed + waiting + active + delayed + prioritized + } + } + } + messageSync { + status + details + } + } + } +`; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/hooks/useGetUptoDateHealthStatus.ts b/packages/twenty-front/src/modules/settings/admin-panel/hooks/useGetUptoDateHealthStatus.ts new file mode 100644 index 000000000..6b04fe53d --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/hooks/useGetUptoDateHealthStatus.ts @@ -0,0 +1,12 @@ +import { useGetSystemHealthStatusQuery } from '~/generated/graphql'; + +export const useGetUptoDateHealthStatus = () => { + const { data, loading } = useGetSystemHealthStatusQuery({ + fetchPolicy: 'network-only', + }); + + return { + healthStatus: data?.getSystemHealthStatus, + healthStatusLoading: loading, + }; +}; diff --git a/packages/twenty-front/src/modules/settings/admin-panel/types/AdminHealthService.ts b/packages/twenty-front/src/modules/settings/admin-panel/types/AdminHealthService.ts new file mode 100644 index 000000000..cb8625157 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/admin-panel/types/AdminHealthService.ts @@ -0,0 +1,12 @@ +import { + AdminPanelHealthServiceData, + AdminPanelWorkerQueueHealth, +} from '~/generated/graphql'; + +type AdminWorkerService = AdminPanelHealthServiceData & { + id: string; + name: string; + queues: AdminPanelWorkerQueueHealth[] | null | undefined; +}; + +export type AdminHealthService = AdminWorkerService; diff --git a/packages/twenty-front/src/modules/types/SettingsPath.ts b/packages/twenty-front/src/modules/types/SettingsPath.ts index 30863792d..6130640ae 100644 --- a/packages/twenty-front/src/modules/types/SettingsPath.ts +++ b/packages/twenty-front/src/modules/types/SettingsPath.ts @@ -34,6 +34,8 @@ export enum SettingsPath { Releases = 'releases', AdminPanel = 'admin-panel', FeatureFlags = 'admin-panel/feature-flags', + AdminPanelHealthStatus = 'admin-panel#health-status', + AdminPanelIndicatorHealthStatus = 'admin-panel/health-status/:indicatorName', Lab = 'lab', Roles = 'roles', RoleDetail = 'roles/:roleId', diff --git a/packages/twenty-front/src/pages/settings/SettingsWorkspace.tsx b/packages/twenty-front/src/pages/settings/SettingsWorkspace.tsx index 807a5de0b..65390d2e7 100644 --- a/packages/twenty-front/src/pages/settings/SettingsWorkspace.tsx +++ b/packages/twenty-front/src/pages/settings/SettingsWorkspace.tsx @@ -1,12 +1,6 @@ import { useLingui } from '@lingui/react/macro'; import { useRecoilValue } from 'recoil'; -import { - GithubVersionLink, - H2Title, - IconWorld, - Section, - UndecoratedLink, -} from 'twenty-ui'; +import { H2Title, IconWorld, Section, UndecoratedLink } from 'twenty-ui'; import { isMultiWorkspaceEnabledState } from '@/client-config/states/isMultiWorkspaceEnabledState'; import { SettingsCard } from '@/settings/components/SettingsCard'; @@ -18,7 +12,6 @@ import { WorkspaceLogoUploader } from '@/settings/workspace/components/Workspace import { SettingsPath } from '@/types/SettingsPath'; import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer'; import { getSettingsPath } from '~/utils/navigation/getSettingsPath'; -import packageJson from '../../../package.json'; export const SettingsWorkspace = () => { const isMultiWorkspaceEnabled = useRecoilValue(isMultiWorkspaceEnabledState); @@ -70,9 +63,6 @@ export const SettingsWorkspace = () => {
-
- -
); diff --git a/packages/twenty-front/src/pages/settings/admin-panel/SettingsAdminIndicatorHealthStatus.tsx b/packages/twenty-front/src/pages/settings/admin-panel/SettingsAdminIndicatorHealthStatus.tsx new file mode 100644 index 000000000..df481742e --- /dev/null +++ b/packages/twenty-front/src/pages/settings/admin-panel/SettingsAdminIndicatorHealthStatus.tsx @@ -0,0 +1,137 @@ +import { SettingsAdminQueueExpandableContainer } from '@/settings/admin-panel/components/SettingsAdminQueueExpandableContainer'; +import { SettingsAdminQueueHealthButtons } from '@/settings/admin-panel/components/SettingsAdminQueueHealthButtons'; +import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer'; +import { SettingsPath } from '@/types/SettingsPath'; +import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer'; +import styled from '@emotion/styled'; +import { useLingui } from '@lingui/react/macro'; +import { useState } from 'react'; +import { useParams } from 'react-router-dom'; +import { H2Title, Section, Status } from 'twenty-ui'; +import { + AdminPanelHealthServiceStatus, + AdminPanelIndicatorHealthStatusInputEnum, + useGetIndicatorHealthStatusQuery, +} from '~/generated/graphql'; +import { getSettingsPath } from '~/utils/navigation/getSettingsPath'; + +const StyledStatusContainer = styled.div``; + +const StyledTitleContainer = styled.div` + display: flex; + flex-direction: column; + flex: 1; +`; + +const StyledErrorMessage = styled.div` + color: ${({ theme }) => theme.color.red}; + margin-top: ${({ theme }) => theme.spacing(2)}; +`; + +const StyledDetailsContainer = styled.pre` + background-color: ${({ theme }) => theme.background.quaternary}; + padding: ${({ theme }) => theme.spacing(6)}; + border-radius: ${({ theme }) => theme.border.radius.sm}; + white-space: pre-wrap; + font-size: ${({ theme }) => theme.font.size.sm}; +`; + +export const SettingsAdminIndicatorHealthStatus = () => { + const { t } = useLingui(); + const { indicatorName } = useParams(); + const { data, loading } = useGetIndicatorHealthStatusQuery({ + variables: { + indicatorName: indicatorName as AdminPanelIndicatorHealthStatusInputEnum, + }, + }); + + const formattedDetails = data?.getIndicatorHealthStatus.details + ? JSON.stringify(JSON.parse(data.getIndicatorHealthStatus.details), null, 2) + : null; + + const isWorkerDown = + !data?.getIndicatorHealthStatus.status || + data?.getIndicatorHealthStatus.status === + AdminPanelHealthServiceStatus.OUTAGE; + + const [selectedQueue, setSelectedQueue] = useState(null); + + const toggleQueueVisibility = (queueName: string) => { + setSelectedQueue(selectedQueue === queueName ? null : queueName); + }; + + return ( + + +
+ + + {data?.getIndicatorHealthStatus.status === + AdminPanelHealthServiceStatus.OPERATIONAL && ( + + )} + {data?.getIndicatorHealthStatus.status === + AdminPanelHealthServiceStatus.OUTAGE && ( + + )} + +
+ + {indicatorName === AdminPanelIndicatorHealthStatusInputEnum.WORKER ? ( +
+ + + + {isWorkerDown && !loading ? ( + + Queue information is not available because the worker is down + + ) : ( + <> + + + + )} +
+ ) : null} + + {indicatorName === AdminPanelIndicatorHealthStatusInputEnum.DATABASE || + indicatorName === AdminPanelIndicatorHealthStatusInputEnum.REDIS ? ( +
+ {formattedDetails && ( + + {formattedDetails} + + )} +
+ ) : null} + + + ); +}; diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts new file mode 100644 index 000000000..3151f81fa --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts @@ -0,0 +1,193 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service'; +import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto'; +import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; +import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health'; +import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; + +describe('AdminPanelHealthService', () => { + let service: AdminPanelHealthService; + let databaseHealth: jest.Mocked; + let redisHealth: jest.Mocked; + let workerHealth: jest.Mocked; + let messageSyncHealth: jest.Mocked; + + beforeEach(async () => { + databaseHealth = { + isHealthy: jest.fn(), + } as any; + + redisHealth = { + isHealthy: jest.fn(), + } as any; + + workerHealth = { + isHealthy: jest.fn(), + } as any; + + messageSyncHealth = { + isHealthy: jest.fn(), + } as any; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + AdminPanelHealthService, + { + provide: DatabaseHealthIndicator, + useValue: databaseHealth, + }, + { + provide: RedisHealthIndicator, + useValue: redisHealth, + }, + { + provide: WorkerHealthIndicator, + useValue: workerHealth, + }, + { + provide: MessageSyncHealthIndicator, + useValue: messageSyncHealth, + }, + ], + }).compile(); + + service = module.get(AdminPanelHealthService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should transform health check response to SystemHealth format', async () => { + databaseHealth.isHealthy.mockResolvedValue({ + database: { + status: 'up', + details: 'Database is healthy', + }, + }); + redisHealth.isHealthy.mockResolvedValue({ + redis: { + status: 'up', + details: 'Redis is connected', + }, + }); + workerHealth.isHealthy.mockResolvedValue({ + worker: { + status: 'up', + queues: [ + { + name: 'test', + workers: 1, + metrics: { + active: 1, + completed: 0, + delayed: 4, + failed: 3, + waiting: 0, + prioritized: 0, + }, + }, + ], + }, + }); + messageSyncHealth.isHealthy.mockResolvedValue({ + messageSync: { + status: 'up', + details: 'Message sync is operational', + }, + }); + + const result = await service.getSystemHealthStatus(); + + const expected: SystemHealth = { + database: { + status: AdminPanelHealthServiceStatus.OPERATIONAL, + details: '"Database is healthy"', + queues: undefined, + }, + redis: { + status: AdminPanelHealthServiceStatus.OPERATIONAL, + details: '"Redis is connected"', + queues: undefined, + }, + worker: { + status: AdminPanelHealthServiceStatus.OPERATIONAL, + details: undefined, + queues: [ + { + name: 'test', + workers: 1, + status: AdminPanelHealthServiceStatus.OPERATIONAL, + metrics: { + active: 1, + completed: 0, + delayed: 4, + failed: 3, + waiting: 0, + prioritized: 0, + }, + }, + ], + }, + messageSync: { + status: AdminPanelHealthServiceStatus.OPERATIONAL, + details: '"Message sync is operational"', + queues: undefined, + }, + }; + + expect(result).toStrictEqual(expected); + }); + + it('should handle mixed health statuses', async () => { + databaseHealth.isHealthy.mockResolvedValue({ + database: { status: 'up' }, + }); + redisHealth.isHealthy.mockRejectedValue( + new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED), + ); + workerHealth.isHealthy.mockResolvedValue({ + worker: { status: 'up', queues: [] }, + }); + messageSyncHealth.isHealthy.mockResolvedValue({ + messageSync: { status: 'up' }, + }); + + const result = await service.getSystemHealthStatus(); + + expect(result).toMatchObject({ + database: { status: AdminPanelHealthServiceStatus.OPERATIONAL }, + redis: { status: AdminPanelHealthServiceStatus.OUTAGE }, + worker: { status: AdminPanelHealthServiceStatus.OPERATIONAL }, + messageSync: { status: AdminPanelHealthServiceStatus.OPERATIONAL }, + }); + }); + + it('should handle all services down', async () => { + databaseHealth.isHealthy.mockRejectedValue( + new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED), + ); + redisHealth.isHealthy.mockRejectedValue( + new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED), + ); + workerHealth.isHealthy.mockRejectedValue( + new Error(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS), + ); + messageSyncHealth.isHealthy.mockRejectedValue( + new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED), + ); + + const result = await service.getSystemHealthStatus(); + + expect(result).toMatchObject({ + database: { status: AdminPanelHealthServiceStatus.OUTAGE }, + redis: { status: AdminPanelHealthServiceStatus.OUTAGE }, + worker: { status: AdminPanelHealthServiceStatus.OUTAGE }, + messageSync: { status: AdminPanelHealthServiceStatus.OUTAGE }, + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel.service.spec.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel.spec.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel.service.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts new file mode 100644 index 000000000..d43792243 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts @@ -0,0 +1,108 @@ +import { Injectable } from '@nestjs/common'; +import { HealthIndicatorResult } from '@nestjs/terminus'; + +import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto'; +import { AdminPanelIndicatorHealthStatusInputEnum } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input'; +import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto'; +import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; +import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health'; +import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; + +@Injectable() +export class AdminPanelHealthService { + constructor( + private readonly databaseHealth: DatabaseHealthIndicator, + private readonly redisHealth: RedisHealthIndicator, + private readonly workerHealth: WorkerHealthIndicator, + private readonly messageSyncHealth: MessageSyncHealthIndicator, + ) {} + + private readonly healthIndicators = { + database: this.databaseHealth, + redis: this.redisHealth, + worker: this.workerHealth, + messageSync: this.messageSyncHealth, + }; + + private getServiceStatus( + result: PromiseSettledResult, + ) { + if (result.status === 'fulfilled') { + const key = Object.keys(result.value)[0]; + const serviceResult = result.value[key]; + const details = serviceResult.details; + + return { + status: + serviceResult.status === 'up' + ? AdminPanelHealthServiceStatus.OPERATIONAL + : AdminPanelHealthServiceStatus.OUTAGE, + details: details ? JSON.stringify(details) : undefined, + queues: serviceResult.queues, + }; + } + + return { + status: AdminPanelHealthServiceStatus.OUTAGE, + details: result.reason?.message, + }; + } + + async getIndicatorHealthStatus( + indicatorName: AdminPanelIndicatorHealthStatusInputEnum, + ): Promise { + const healthIndicator = this.healthIndicators[indicatorName]; + + if (!healthIndicator) { + throw new Error(`Health indicator not found: ${indicatorName}`); + } + + const result = await Promise.allSettled([healthIndicator.isHealthy()]); + const indicatorStatus = this.getServiceStatus(result[0]); + + if (indicatorName === 'worker') { + return { + ...indicatorStatus, + queues: (indicatorStatus?.queues ?? []).map((queue) => ({ + ...queue, + status: + queue.workers > 0 + ? AdminPanelHealthServiceStatus.OPERATIONAL + : AdminPanelHealthServiceStatus.OUTAGE, + })), + }; + } + + return indicatorStatus; + } + + async getSystemHealthStatus(): Promise { + const [databaseResult, redisResult, workerResult, messageSyncResult] = + await Promise.allSettled([ + this.databaseHealth.isHealthy(), + this.redisHealth.isHealthy(), + this.workerHealth.isHealthy(), + this.messageSyncHealth.isHealthy(), + ]); + + const workerStatus = this.getServiceStatus(workerResult); + + return { + database: this.getServiceStatus(databaseResult), + redis: this.getServiceStatus(redisResult), + worker: { + ...workerStatus, + queues: (workerStatus?.queues ?? []).map((queue) => ({ + ...queue, + status: + queue.workers > 0 + ? AdminPanelHealthServiceStatus.OPERATIONAL + : AdminPanelHealthServiceStatus.OUTAGE, + })), + }, + messageSync: this.getServiceStatus(messageSyncResult), + }; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts index 5e929e10a..6defbcb60 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts @@ -1,21 +1,28 @@ import { Module } from '@nestjs/common'; +import { TerminusModule } from '@nestjs/terminus'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service'; import { AdminPanelResolver } from 'src/engine/core-modules/admin-panel/admin-panel.resolver'; import { AdminPanelService } from 'src/engine/core-modules/admin-panel/admin-panel.service'; import { AuthModule } from 'src/engine/core-modules/auth/auth.module'; +import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module'; import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; +import { HealthModule } from 'src/engine/core-modules/health/health.module'; +import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module'; import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; -import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module'; @Module({ imports: [ TypeOrmModule.forFeature([User, Workspace, FeatureFlag], 'core'), AuthModule, DomainManagerModule, + HealthModule, + RedisClientModule, + TerminusModule, ], - providers: [AdminPanelResolver, AdminPanelService], + providers: [AdminPanelResolver, AdminPanelService, AdminPanelHealthService], exports: [AdminPanelService], }) export class AdminPanelModule {} diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts index 4a15dbc0f..cab4622d5 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts @@ -1,10 +1,12 @@ import { UseFilters, UseGuards } from '@nestjs/common'; import { Args, Mutation, Query, Resolver } from '@nestjs/graphql'; +import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service'; import { AdminPanelService } from 'src/engine/core-modules/admin-panel/admin-panel.service'; import { EnvironmentVariablesOutput } from 'src/engine/core-modules/admin-panel/dtos/environment-variables.output'; import { ImpersonateInput } from 'src/engine/core-modules/admin-panel/dtos/impersonate.input'; import { ImpersonateOutput } from 'src/engine/core-modules/admin-panel/dtos/impersonate.output'; +import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto'; import { UpdateWorkspaceFeatureFlagInput } from 'src/engine/core-modules/admin-panel/dtos/update-workspace-feature-flag.input'; import { UserLookup } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.entity'; import { UserLookupInput } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.input'; @@ -13,10 +15,16 @@ import { ImpersonateGuard } from 'src/engine/guards/impersonate-guard'; import { UserAuthGuard } from 'src/engine/guards/user-auth.guard'; import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; +import { AdminPanelHealthServiceData } from './dtos/admin-panel-health-service-data.dto'; +import { AdminPanelIndicatorHealthStatusInputEnum } from './dtos/admin-panel-indicator-health-status.input'; + @Resolver() @UseFilters(AuthGraphqlApiExceptionFilter) export class AdminPanelResolver { - constructor(private adminService: AdminPanelService) {} + constructor( + private adminService: AdminPanelService, + private adminPanelHealthService: AdminPanelHealthService, + ) {} @UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard) @Mutation(() => ImpersonateOutput) @@ -53,4 +61,20 @@ export class AdminPanelResolver { async getEnvironmentVariablesGrouped(): Promise { return this.adminService.getEnvironmentVariablesGrouped(); } + + @UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard) + @Query(() => SystemHealth) + async getSystemHealthStatus(): Promise { + return this.adminPanelHealthService.getSystemHealthStatus(); + } + + @Query(() => AdminPanelHealthServiceData) + async getIndicatorHealthStatus( + @Args('indicatorName', { + type: () => AdminPanelIndicatorHealthStatusInputEnum, + }) + indicatorName: AdminPanelIndicatorHealthStatusInputEnum, + ): Promise { + return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorName); + } } diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts new file mode 100644 index 000000000..9d1699f4c --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts @@ -0,0 +1,16 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +import { AdminPanelWorkerQueueHealth } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto'; +import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; + +@ObjectType() +export class AdminPanelHealthServiceData { + @Field(() => AdminPanelHealthServiceStatus) + status: AdminPanelHealthServiceStatus; + + @Field(() => String, { nullable: true }) + details?: string; + + @Field(() => [AdminPanelWorkerQueueHealth], { nullable: true }) + queues?: AdminPanelWorkerQueueHealth[]; +} diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input.ts new file mode 100644 index 000000000..c17accd2a --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input.ts @@ -0,0 +1,17 @@ +import { Field, registerEnumType } from '@nestjs/graphql'; + +export enum AdminPanelIndicatorHealthStatusInputEnum { + DATABASE = 'database', + REDIS = 'redis', + WORKER = 'worker', + MESSAGE_SYNC = 'messageSync', +} + +registerEnumType(AdminPanelIndicatorHealthStatusInputEnum, { + name: 'AdminPanelIndicatorHealthStatusInputEnum', +}); + +export class AdminPanelIndicatorHealthStatusInput { + @Field(() => AdminPanelIndicatorHealthStatusInputEnum) + indicatorName: AdminPanelIndicatorHealthStatusInputEnum; +} diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto.ts new file mode 100644 index 000000000..9d15813ca --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto.ts @@ -0,0 +1,10 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; +import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type'; + +@ObjectType() +export class AdminPanelWorkerQueueHealth extends WorkerQueueHealth { + @Field(() => AdminPanelHealthServiceStatus) + status: AdminPanelHealthServiceStatus; +} diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts new file mode 100644 index 000000000..96a726eb4 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts @@ -0,0 +1,18 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto'; + +@ObjectType() +export class SystemHealth { + @Field(() => AdminPanelHealthServiceData) + database: AdminPanelHealthServiceData; + + @Field(() => AdminPanelHealthServiceData) + redis: AdminPanelHealthServiceData; + + @Field(() => AdminPanelHealthServiceData) + worker: AdminPanelHealthServiceData; + + @Field(() => AdminPanelHealthServiceData) + messageSync: AdminPanelHealthServiceData; +} diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum.ts new file mode 100644 index 000000000..7298e7618 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum.ts @@ -0,0 +1,10 @@ +import { registerEnumType } from '@nestjs/graphql'; + +export enum AdminPanelHealthServiceStatus { + OPERATIONAL = 'operational', + OUTAGE = 'outage', +} + +registerEnumType(AdminPanelHealthServiceStatus, { + name: 'AdminPanelHealthServiceStatus', +}); diff --git a/packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts b/packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts new file mode 100644 index 000000000..1996ddd09 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts @@ -0,0 +1,12 @@ +export const HEALTH_ERROR_MESSAGES = { + NO_ACTIVE_WORKERS: 'No active workers found', + WORKER_TIMEOUT: 'Worker check timeout', + DATABASE_TIMEOUT: 'Database timeout', + REDIS_TIMEOUT: 'Redis timeout', + DATABASE_CONNECTION_FAILED: 'Database connection failed', + REDIS_CONNECTION_FAILED: 'Unknown Redis error', + WORKER_CHECK_FAILED: 'Worker check failed', + MESSAGE_SYNC_TIMEOUT: 'Message sync check timeout', + MESSAGE_SYNC_CHECK_FAILED: 'Message sync check failed', + MESSAGE_SYNC_HIGH_FAILURE_RATE: 'High failure rate in message sync jobs', +} as const; diff --git a/packages/twenty-server/src/engine/core-modules/health/constants/health-indicators-timeout.conts.ts b/packages/twenty-server/src/engine/core-modules/health/constants/health-indicators-timeout.conts.ts new file mode 100644 index 000000000..c0def54c7 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/constants/health-indicators-timeout.conts.ts @@ -0,0 +1 @@ +export const HEALTH_INDICATORS_TIMEOUT = 3000; diff --git a/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts new file mode 100644 index 000000000..6e4012188 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts @@ -0,0 +1,41 @@ +import { HealthCheckService } from '@nestjs/terminus'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; +import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; + +describe('HealthController', () => { + let healthController: HealthController; + + beforeEach(async () => { + const testingModule: TestingModule = await Test.createTestingModule({ + controllers: [HealthController], + providers: [ + { + provide: HealthCheckService, + useValue: { check: jest.fn() }, + }, + { + provide: DatabaseHealthIndicator, + useValue: { isHealthy: jest.fn() }, + }, + { + provide: RedisHealthIndicator, + useValue: { isHealthy: jest.fn() }, + }, + { + provide: WorkerHealthIndicator, + useValue: { isHealthy: jest.fn() }, + }, + ], + }).compile(); + + healthController = testingModule.get(HealthController); + }); + + it('should be defined', () => { + expect(healthController).toBeDefined(); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts new file mode 100644 index 000000000..0eb5fb167 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts @@ -0,0 +1,29 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller'; +import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; + +describe('MetricsController', () => { + let metricsController: MetricsController; + + beforeEach(async () => { + const testingModule: TestingModule = await Test.createTestingModule({ + controllers: [MetricsController], + providers: [ + { + provide: HealthCacheService, + useValue: { + getMessageChannelSyncJobByStatusCounter: jest.fn(), + getInvalidCaptchaCounter: jest.fn(), + }, + }, + ], + }).compile(); + + metricsController = testingModule.get(MetricsController); + }); + + it('should be defined', () => { + expect(metricsController).toBeDefined(); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts new file mode 100644 index 000000000..084aa6b2f --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts @@ -0,0 +1,39 @@ +import { BadRequestException, Controller, Get, Param } from '@nestjs/common'; +import { HealthCheck, HealthCheckService } from '@nestjs/terminus'; + +import { HealthServiceName } from 'src/engine/core-modules/health/enums/health-service-name.enum'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; +import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; + +@Controller('healthz') +export class HealthController { + constructor( + private readonly health: HealthCheckService, + private readonly databaseHealth: DatabaseHealthIndicator, + private readonly redisHealth: RedisHealthIndicator, + private readonly workerHealth: WorkerHealthIndicator, + ) {} + + @Get() + @HealthCheck() + check() { + return this.health.check([]); + } + + @Get('/:serviceName') + @HealthCheck() + checkService(@Param('serviceName') serviceName: HealthServiceName) { + const checks = { + [HealthServiceName.DATABASE]: () => this.databaseHealth.isHealthy(), + [HealthServiceName.REDIS]: () => this.redisHealth.isHealthy(), + [HealthServiceName.WORKER]: () => this.workerHealth.isHealthy(), + }; + + if (!(serviceName in checks)) { + throw new BadRequestException(`Invalid service name: ${serviceName}`); + } + + return this.health.check([checks[serviceName]]); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/health/health.controller.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts similarity index 59% rename from packages/twenty-server/src/engine/core-modules/health/health.controller.ts rename to packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts index 50c4f034a..91ac62aac 100644 --- a/packages/twenty-server/src/engine/core-modules/health/health.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts @@ -1,20 +1,10 @@ import { Controller, Get } from '@nestjs/common'; -import { HealthCheck, HealthCheckService } from '@nestjs/terminus'; import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; -@Controller('healthz') -export class HealthController { - constructor( - private health: HealthCheckService, - private healthCacheService: HealthCacheService, - ) {} - - @Get() - @HealthCheck() - check() { - return this.health.check([]); - } +@Controller('metricsz') +export class MetricsController { + constructor(private readonly healthCacheService: HealthCacheService) {} @Get('/message-channel-sync-job-by-status-counter') getMessageChannelSyncJobByStatusCounter() { diff --git a/packages/twenty-server/src/engine/core-modules/health/enums/health-service-name.enum.ts b/packages/twenty-server/src/engine/core-modules/health/enums/health-service-name.enum.ts new file mode 100644 index 000000000..e00e57368 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/enums/health-service-name.enum.ts @@ -0,0 +1,6 @@ +export enum HealthServiceName { + DATABASE = 'database', + REDIS = 'redis', + WORKER = 'worker', + MESSAGE_SYNC = 'messageSync', +} diff --git a/packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts b/packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts index 39f907d13..ec03522b5 100644 --- a/packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts @@ -5,7 +5,7 @@ import { CacheStorageService } from 'src/engine/core-modules/cache-storage/servi import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type'; -import { MessageChannelSyncJobByStatusCounter } from 'src/engine/core-modules/health/types/health-metrics.types'; +import { MessageChannelSyncJobByStatusCounter } from 'src/engine/core-modules/health/types/message-sync-metrics.types'; import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @Injectable() diff --git a/packages/twenty-server/src/engine/core-modules/health/health.controller.spec.ts b/packages/twenty-server/src/engine/core-modules/health/health.controller.spec.ts deleted file mode 100644 index 2f863e3af..000000000 --- a/packages/twenty-server/src/engine/core-modules/health/health.controller.spec.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { HealthCheckService, HttpHealthIndicator } from '@nestjs/terminus'; -import { Test, TestingModule } from '@nestjs/testing'; - -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; -import { HealthController } from 'src/engine/core-modules/health/health.controller'; - -describe('HealthController', () => { - let healthController: HealthController; - let testingModule: TestingModule; - - beforeEach(async () => { - testingModule = await Test.createTestingModule({ - providers: [ - HealthController, - { - provide: HealthCheckService, - useValue: {}, - }, - { - provide: HttpHealthIndicator, - useValue: {}, - }, - { - provide: HealthCacheService, - useValue: {}, - }, - ], - }).compile(); - - healthController = testingModule.get(HealthController); - }); - - it('should be defined', () => { - expect(healthController).toBeDefined(); - }); -}); diff --git a/packages/twenty-server/src/engine/core-modules/health/health.module.ts b/packages/twenty-server/src/engine/core-modules/health/health.module.ts index 1958e50be..3eff6391a 100644 --- a/packages/twenty-server/src/engine/core-modules/health/health.module.ts +++ b/packages/twenty-server/src/engine/core-modules/health/health.module.ts @@ -1,13 +1,32 @@ import { Module } from '@nestjs/common'; import { TerminusModule } from '@nestjs/terminus'; -import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; -import { HealthController } from 'src/engine/core-modules/health/health.controller'; +import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller'; +import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller'; +import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health'; +import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module'; +import { HealthCacheService } from './health-cache.service'; + +import { DatabaseHealthIndicator } from './indicators/database.health'; +import { RedisHealthIndicator } from './indicators/redis.health'; +import { WorkerHealthIndicator } from './indicators/worker.health'; @Module({ - imports: [TerminusModule], - controllers: [HealthController], - providers: [HealthCacheService], - exports: [HealthCacheService], + imports: [TerminusModule, RedisClientModule], + controllers: [HealthController, MetricsController], + providers: [ + HealthCacheService, + DatabaseHealthIndicator, + RedisHealthIndicator, + WorkerHealthIndicator, + MessageSyncHealthIndicator, + ], + exports: [ + HealthCacheService, + DatabaseHealthIndicator, + RedisHealthIndicator, + WorkerHealthIndicator, + MessageSyncHealthIndicator, + ], }) export class HealthModule {} diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts new file mode 100644 index 000000000..5d1c853a6 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts @@ -0,0 +1,116 @@ +import { HealthIndicatorService } from '@nestjs/terminus'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { DataSource } from 'typeorm'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; + +describe('DatabaseHealthIndicator', () => { + let service: DatabaseHealthIndicator; + let dataSource: jest.Mocked; + let healthIndicatorService: jest.Mocked; + + beforeEach(async () => { + dataSource = { + query: jest.fn(), + } as any; + + healthIndicatorService = { + check: jest.fn().mockReturnValue({ + up: jest.fn().mockImplementation((data) => ({ + database: { status: 'up', ...data }, + })), + down: jest.fn().mockImplementation((error) => ({ + database: { + status: 'down', + error, + }, + })), + }), + } as any; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DatabaseHealthIndicator, + { + provide: 'coreDataSource', + useValue: dataSource, + }, + { + provide: HealthIndicatorService, + useValue: healthIndicatorService, + }, + ], + }).compile(); + + service = module.get(DatabaseHealthIndicator); + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should return up status with details when database responds', async () => { + const mockResponses = [ + [{ version: 'PostgreSQL 15.6' }], + [{ count: '5' }], + [{ max_connections: '100' }], + [{ uptime: '3600' }], + [{ size: '1 GB' }], + [{ table_stats: [] }], + [{ ratio: '95.5' }], + [{ deadlocks: '0' }], + [{ count: '0' }], + ]; + + mockResponses.forEach((response) => { + dataSource.query.mockResolvedValueOnce(response); + }); + + const result = await service.isHealthy(); + + expect(result.database.status).toBe('up'); + expect(result.database.details).toBeDefined(); + expect(result.database.details.version).toBeDefined(); + expect(result.database.details.connections).toBeDefined(); + expect(result.database.details.performance).toBeDefined(); + }); + + it('should return down status when database fails', async () => { + dataSource.query.mockRejectedValueOnce( + new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED), + ); + + const result = await service.isHealthy(); + + expect(result.database.status).toBe('down'); + expect(result.database.error).toBe( + HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED, + ); + }); + + it('should timeout after specified duration', async () => { + dataSource.query.mockImplementationOnce( + () => + new Promise((resolve) => + setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100), + ), + ); + + const healthCheckPromise = service.isHealthy(); + + jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1); + + const result = await healthCheckPromise; + + expect(result.database.status).toBe('down'); + expect(result.database.error).toBe(HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/message-sync.health.spec.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/message-sync.health.spec.ts new file mode 100644 index 000000000..e76058b7e --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/message-sync.health.spec.ts @@ -0,0 +1,137 @@ +import { HealthIndicatorService } from '@nestjs/terminus'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; +import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; +import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health'; +import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; + +describe('MessageSyncHealthIndicator', () => { + let service: MessageSyncHealthIndicator; + let healthCacheService: jest.Mocked; + let healthIndicatorService: jest.Mocked; + + beforeEach(async () => { + healthCacheService = { + getMessageChannelSyncJobByStatusCounter: jest.fn(), + } as any; + + healthIndicatorService = { + check: jest.fn().mockReturnValue({ + up: jest.fn().mockImplementation((data) => ({ + messageSync: { status: 'up', ...data }, + })), + down: jest.fn().mockImplementation((error) => ({ + messageSync: { status: 'down', error }, + })), + }), + } as any; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + MessageSyncHealthIndicator, + { + provide: HealthCacheService, + useValue: healthCacheService, + }, + { + provide: HealthIndicatorService, + useValue: healthIndicatorService, + }, + ], + }).compile(); + + service = module.get( + MessageSyncHealthIndicator, + ); + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should return up status when no jobs are present', async () => { + healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue( + { + [MessageChannelSyncStatus.NOT_SYNCED]: 0, + [MessageChannelSyncStatus.ONGOING]: 0, + [MessageChannelSyncStatus.ACTIVE]: 0, + [MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0, + [MessageChannelSyncStatus.FAILED_UNKNOWN]: 0, + }, + ); + + const result = await service.isHealthy(); + + expect(result.messageSync.status).toBe('up'); + expect(result.messageSync.details.totalJobs).toBe(0); + expect(result.messageSync.details.failedJobs).toBe(0); + expect(result.messageSync.details.failureRate).toBe(0); + }); + + it('should return up status when failure rate is below 20%', async () => { + healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue( + { + [MessageChannelSyncStatus.NOT_SYNCED]: 0, + [MessageChannelSyncStatus.ONGOING]: 2, + [MessageChannelSyncStatus.ACTIVE]: 8, + [MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0, + [MessageChannelSyncStatus.FAILED_UNKNOWN]: 1, + }, + ); + + const result = await service.isHealthy(); + + expect(result.messageSync.status).toBe('up'); + expect(result.messageSync.details.totalJobs).toBe(11); + expect(result.messageSync.details.failedJobs).toBe(1); + expect(result.messageSync.details.failureRate).toBe(9.09); + }); + + it('should return down status when failure rate is above 20%', async () => { + healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue( + { + [MessageChannelSyncStatus.NOT_SYNCED]: 0, + [MessageChannelSyncStatus.ONGOING]: 1, + [MessageChannelSyncStatus.ACTIVE]: 1, + [MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2, + [MessageChannelSyncStatus.FAILED_UNKNOWN]: 2, + }, + ); + + const result = await service.isHealthy(); + + expect(result.messageSync.status).toBe('down'); + expect(result.messageSync.error.error).toBe( + HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE, + ); + expect(result.messageSync.error.details).toBeDefined(); + expect(result.messageSync.error.details.failureRate).toBe(33.33); + }); + + it('should timeout after specified duration', async () => { + healthCacheService.getMessageChannelSyncJobByStatusCounter.mockImplementationOnce( + () => + new Promise((resolve) => + setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100), + ), + ); + + const healthCheckPromise = service.isHealthy(); + + jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1); + + const result = await healthCheckPromise; + + expect(result.messageSync.status).toBe('down'); + expect(result.messageSync.error).toBe( + HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT, + ); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts new file mode 100644 index 000000000..09d750c68 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts @@ -0,0 +1,133 @@ +import { HealthIndicatorService } from '@nestjs/terminus'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { Redis } from 'ioredis'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; +import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; + +describe('RedisHealthIndicator', () => { + let service: RedisHealthIndicator; + let mockRedis: jest.Mocked< + Pick + >; + let healthIndicatorService: jest.Mocked; + + beforeEach(async () => { + mockRedis = { + ping: jest.fn(), + info: jest.fn(), + dbsize: jest.fn(), + memory: jest.fn(), + }; + + const mockRedisService = { + getClient: () => mockRedis, + } as unknown as RedisClientService; + + healthIndicatorService = { + check: jest.fn().mockReturnValue({ + up: jest.fn().mockImplementation((data) => ({ + redis: { status: 'up', ...data }, + })), + down: jest.fn().mockImplementation((error) => ({ + redis: { + status: 'down', + error, + }, + })), + }), + } as any; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + RedisHealthIndicator, + { + provide: RedisClientService, + useValue: mockRedisService, + }, + { + provide: HealthIndicatorService, + useValue: healthIndicatorService, + }, + ], + }).compile(); + + service = module.get(RedisHealthIndicator); + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should return up status with details when redis responds', async () => { + // ai generated mock + mockRedis.info + .mockResolvedValueOnce('redis_version:7.0.0\r\n') + .mockResolvedValueOnce( + 'used_memory_human:1.2G\nused_memory_peak_human:1.5G\nmem_fragmentation_ratio:1.5\n', + ) + .mockResolvedValueOnce('connected_clients:5\n') + .mockResolvedValueOnce( + 'total_connections_received:100\nkeyspace_hits:90\nkeyspace_misses:10\n', + ); + + const result = await service.isHealthy(); + + expect(result.redis.status).toBe('up'); + expect(result.redis.details).toBeDefined(); + expect(result.redis.details.version).toBe('7.0.0'); + }); + + it('should return down status when redis fails', async () => { + mockRedis.ping.mockRejectedValueOnce( + new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED), + ); + + const result = await service.isHealthy(); + + expect(result.redis.status).toBe('down'); + expect(result.redis.error).toBe( + HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED, + ); + }); + + it('should timeout after specified duration', async () => { + mockRedis.ping.mockImplementationOnce( + () => + new Promise((resolve) => + setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100), + ), + ); + + const healthCheckPromise = service.isHealthy(); + + jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1); + + const result = await healthCheckPromise; + + expect(result.redis.status).toBe('down'); + expect(result.redis.error).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT); + }); + + it('should handle partial failures in health details collection', async () => { + mockRedis.info + .mockResolvedValueOnce('redis_version:7.0.0') // info + .mockResolvedValueOnce('used_memory_human:1.2G') // memory + .mockResolvedValueOnce('connected_clients:5') // clients + .mockResolvedValueOnce('total_connections_received:100'); // stats + + const result = await service.isHealthy(); + + expect(result.redis.status).toBe('up'); + expect(result.redis.details).toBeDefined(); + expect(result.redis.details.version).toBe('7.0.0'); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/worker.health.spec.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/worker.health.spec.ts new file mode 100644 index 000000000..0c7f180c1 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/worker.health.spec.ts @@ -0,0 +1,136 @@ +import { HealthIndicatorService } from '@nestjs/terminus'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { Redis } from 'ioredis'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; +import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; + +const mockQueueInstance = { + getWorkers: jest.fn().mockResolvedValue([]), + close: jest.fn().mockResolvedValue(undefined), + getFailedCount: jest.fn().mockResolvedValue(0), + getCompletedCount: jest.fn().mockResolvedValue(0), + getWaitingCount: jest.fn().mockResolvedValue(0), + getActiveCount: jest.fn().mockResolvedValue(0), + getDelayedCount: jest.fn().mockResolvedValue(0), + getPrioritizedCount: jest.fn().mockResolvedValue(0), +}; + +jest.mock('bullmq', () => ({ + Queue: jest.fn(() => mockQueueInstance), +})); + +describe('WorkerHealthIndicator', () => { + let service: WorkerHealthIndicator; + let mockRedis: jest.Mocked>; + let healthIndicatorService: jest.Mocked; + + beforeEach(async () => { + mockRedis = { + ping: jest.fn(), + }; + + const mockRedisService = { + getClient: () => mockRedis, + } as unknown as RedisClientService; + + healthIndicatorService = { + check: jest.fn().mockReturnValue({ + up: jest.fn().mockImplementation((data) => ({ + worker: { status: 'up', ...data }, + })), + down: jest.fn().mockImplementation((error) => ({ + worker: { status: 'down', error }, + })), + }), + } as any; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + WorkerHealthIndicator, + { + provide: RedisClientService, + useValue: mockRedisService, + }, + { + provide: HealthIndicatorService, + useValue: healthIndicatorService, + }, + ], + }).compile(); + + service = module.get(WorkerHealthIndicator); + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should return up status when workers are active', async () => { + mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]); + + const result = await service.isHealthy(); + + expect(result.worker.status).toBe('up'); + expect('queues' in result.worker).toBe(true); + if ('queues' in result.worker) { + expect(result.worker.queues.length).toBeGreaterThan(0); + } + }); + + it('should return down status when no workers are active', async () => { + mockQueueInstance.getWorkers.mockResolvedValue([]); + + const result = await service.isHealthy(); + + expect(result.worker.status).toBe('down'); + expect('error' in result.worker).toBe(true); + if ('error' in result.worker) { + expect(result.worker.error).toBe(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS); + } + }); + + it('should timeout after specified duration', async () => { + jest.useFakeTimers(); + mockQueueInstance.getWorkers.mockImplementationOnce( + () => + new Promise((resolve) => + setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100), + ), + ); + + const resultPromise = service.isHealthy(); + + jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 200); + const result = await resultPromise; + + expect(result.worker.status).toBe('down'); + expect('error' in result.worker).toBe(true); + if ('error' in result.worker) { + expect(result.worker.error).toBe(HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT); + } + jest.useRealTimers(); + }); + + it('should check all message queues', async () => { + mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]); + + await service.isHealthy(); + + expect(mockQueueInstance.getWorkers).toHaveBeenCalledTimes( + Object.keys(MessageQueue).length, + ); + expect(mockQueueInstance.close).toHaveBeenCalledTimes( + Object.keys(MessageQueue).length, + ); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts new file mode 100644 index 000000000..a8cc91db1 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts @@ -0,0 +1,103 @@ +import { Injectable } from '@nestjs/common'; +import { + HealthIndicatorResult, + HealthIndicatorService, +} from '@nestjs/terminus'; +import { InjectDataSource } from '@nestjs/typeorm'; + +import { DataSource } from 'typeorm'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; + +@Injectable() +export class DatabaseHealthIndicator { + constructor( + @InjectDataSource('core') + private readonly dataSource: DataSource, + private readonly healthIndicatorService: HealthIndicatorService, + ) {} + + async isHealthy(): Promise { + const indicator = this.healthIndicatorService.check('database'); + + try { + const [ + [versionResult], + [activeConnections], + [maxConnections], + [uptime], + [databaseSize], + tableStats, + [cacheHitRatio], + [deadlocks], + [slowQueries], + ] = await withHealthCheckTimeout( + Promise.all([ + this.dataSource.query('SELECT version()'), + this.dataSource.query( + 'SELECT count(*) as count FROM pg_stat_activity', + ), + this.dataSource.query('SHOW max_connections'), + this.dataSource.query( + 'SELECT extract(epoch from current_timestamp - pg_postmaster_start_time()) as uptime', + ), + this.dataSource.query( + 'SELECT pg_size_pretty(pg_database_size(current_database())) as size', + ), + this.dataSource.query(` + SELECT schemaname, relname, n_live_tup, n_dead_tup, last_vacuum, last_autovacuum + FROM pg_stat_user_tables + ORDER BY n_live_tup DESC + LIMIT 10 + `), + this.dataSource.query(` + SELECT + sum(heap_blks_hit) * 100.0 / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio + FROM pg_statio_user_tables + `), + this.dataSource.query( + 'SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()', + ), + this.dataSource.query(` + SELECT count(*) as count + FROM pg_stat_activity + WHERE state = 'active' + AND query_start < now() - interval '1 minute' + `), + ]), + HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT, + ); + + return indicator.up({ + details: { + version: versionResult.version, + connections: { + active: parseInt(activeConnections.count), + max: parseInt(maxConnections.max_connections), + utilizationPercent: Math.round( + (parseInt(activeConnections.count) / + parseInt(maxConnections.max_connections)) * + 100, + ), + }, + uptime: Math.round(uptime.uptime / 3600) + ' hours', + databaseSize: databaseSize.size, + performance: { + cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%', + deadlocks: parseInt(deadlocks.deadlocks), + slowQueries: parseInt(slowQueries.count), + }, + top10Tables: tableStats, + }, + }); + } catch (error) { + const errorMessage = + error.message === HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT + ? HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT + : HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED; + + return indicator.down(errorMessage); + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/message-sync.health.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/message-sync.health.ts new file mode 100644 index 000000000..b25e0c022 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/message-sync.health.ts @@ -0,0 +1,63 @@ +import { Injectable } from '@nestjs/common'; +import { + HealthIndicatorResult, + HealthIndicatorService, +} from '@nestjs/terminus'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; + +@Injectable() +export class MessageSyncHealthIndicator { + constructor( + private readonly healthIndicatorService: HealthIndicatorService, + private readonly healthCacheService: HealthCacheService, + ) {} + + async isHealthy(): Promise { + const indicator = this.healthIndicatorService.check('messageSync'); + + try { + const counters = await withHealthCheckTimeout( + this.healthCacheService.getMessageChannelSyncJobByStatusCounter(), + HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT, + ); + + const totalJobs = Object.values(counters).reduce( + (sum, count) => sum + (count || 0), + 0, + ); + + const failedJobs = counters.FAILED_UNKNOWN || 0; + // + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0) + + const failureRate = + totalJobs > 0 + ? Math.round((failedJobs / totalJobs) * 100 * 100) / 100 + : 0; + const details = { + counters, + totalJobs, + failedJobs, + failureRate, + }; + + if (totalJobs === 0 || failureRate < 20) { + return indicator.up({ details }); + } + + return indicator.down({ + error: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE, + details, + }); + } catch (error) { + const errorMessage = + error.message === HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT + ? HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT + : HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED; + + return indicator.down(errorMessage); + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts new file mode 100644 index 000000000..7078f517d --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts @@ -0,0 +1,94 @@ +import { Injectable } from '@nestjs/common'; +import { + HealthIndicatorResult, + HealthIndicatorService, +} from '@nestjs/terminus'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; + +@Injectable() +export class RedisHealthIndicator { + constructor( + private readonly redisClient: RedisClientService, + private readonly healthIndicatorService: HealthIndicatorService, + ) {} + + async isHealthy(): Promise { + const indicator = this.healthIndicatorService.check('redis'); + + try { + const [info, memory, clients, stats] = await withHealthCheckTimeout( + Promise.all([ + this.redisClient.getClient().info(), + this.redisClient.getClient().info('memory'), + this.redisClient.getClient().info('clients'), + this.redisClient.getClient().info('stats'), + ]), + HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT, + ); + + const parseInfo = (info: string) => { + const result: Record = {}; + + info.split('\r\n').forEach((line) => { + const [key, value] = line.split(':'); + + if (key && value) { + result[key] = value; + } + }); + + return result; + }; + + const infoData = parseInfo(info); + const memoryData = parseInfo(memory); + const clientsData = parseInfo(clients); + const statsData = parseInfo(stats); + + return indicator.up({ + details: { + version: infoData.redis_version, + uptime: + Math.round(parseInt(infoData.uptime_in_seconds) / 3600) + ' hours', + memory: { + used: memoryData.used_memory_human, + peak: memoryData.used_memory_peak_human, + fragmentation: parseFloat(memoryData.mem_fragmentation_ratio), + }, + connections: { + current: parseInt(clientsData.connected_clients), + total: parseInt(statsData.total_connections_received), + rejected: parseInt(statsData.rejected_connections), + }, + performance: { + opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec), + hitRate: statsData.keyspace_hits + ? Math.round( + (parseInt(statsData.keyspace_hits) / + (parseInt(statsData.keyspace_hits) + + parseInt(statsData.keyspace_misses))) * + 100, + ) + '%' + : '0%', + evictedKeys: parseInt(statsData.evicted_keys), + expiredKeys: parseInt(statsData.expired_keys), + }, + replication: { + role: infoData.role, + connectedSlaves: parseInt(infoData.connected_slaves || '0'), + }, + }, + }); + } catch (error) { + const errorMessage = + error.message === HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT + ? HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT + : HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED; + + return indicator.down(errorMessage); + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts b/packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts new file mode 100644 index 000000000..4e47607fd --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts @@ -0,0 +1,103 @@ +import { Injectable } from '@nestjs/common'; +import { HealthIndicatorService } from '@nestjs/terminus'; + +import { Queue } from 'bullmq'; + +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; +import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; +import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; +import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; + +@Injectable() +export class WorkerHealthIndicator { + constructor( + private readonly redisClient: RedisClientService, + private readonly healthIndicatorService: HealthIndicatorService, + ) {} + + async isHealthy() { + const indicator = this.healthIndicatorService.check('worker'); + + try { + const workerStatus = await withHealthCheckTimeout( + this.checkWorkers(), + HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT, + ); + + if (workerStatus.status === 'up') { + return indicator.up({ + queues: workerStatus.queues, + }); + } + + return indicator.down(workerStatus.error); + } catch (error) { + const errorMessage = + error.message === HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT + ? HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT + : HEALTH_ERROR_MESSAGES.WORKER_CHECK_FAILED; + + return indicator.down(errorMessage); + } + } + + private async checkWorkers() { + const redis = this.redisClient.getClient(); + const queues = Object.values(MessageQueue); + const queueStatuses: WorkerQueueHealth[] = []; + + for (const queueName of queues) { + const queue = new Queue(queueName, { connection: redis }); + + try { + const workers = await queue.getWorkers(); + + if (workers.length > 0) { + const [ + failedCount, + completedCount, + waitingCount, + activeCount, + delayedCount, + prioritizedCount, + ] = await Promise.all([ + queue.getFailedCount(), + queue.getCompletedCount(), + queue.getWaitingCount(), + queue.getActiveCount(), + queue.getDelayedCount(), + queue.getPrioritizedCount(), + ]); + + queueStatuses.push({ + name: queueName, + workers: workers.length, + metrics: { + failed: failedCount, + completed: completedCount, + waiting: waitingCount, + active: activeCount, + delayed: delayedCount, + prioritized: prioritizedCount, + }, + }); + } + + await queue.close(); + } catch (error) { + await queue.close(); + } + } + + const hasActiveWorkers = queueStatuses.some((q) => q.workers > 0); + + return { + status: hasActiveWorkers ? 'up' : 'down', + error: hasActiveWorkers + ? undefined + : HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS, + queues: queueStatuses, + }; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/health/types/health-metrics.types.ts b/packages/twenty-server/src/engine/core-modules/health/types/health-metrics.types.ts deleted file mode 100644 index 5c2ae54b3..000000000 --- a/packages/twenty-server/src/engine/core-modules/health/types/health-metrics.types.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; - -export type MessageChannelSyncJobByStatusCounter = { - [key in MessageChannelSyncStatus]?: number; -}; diff --git a/packages/twenty-server/src/engine/core-modules/health/types/message-sync-metrics.types.ts b/packages/twenty-server/src/engine/core-modules/health/types/message-sync-metrics.types.ts new file mode 100644 index 000000000..0562f93ad --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/types/message-sync-metrics.types.ts @@ -0,0 +1,19 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +@ObjectType() +export class MessageChannelSyncJobByStatusCounter { + @Field(() => Number, { nullable: true }) + NOT_SYNCED?: number; + + @Field(() => Number, { nullable: true }) + ONGOING?: number; + + @Field(() => Number, { nullable: true }) + ACTIVE?: number; + + @Field(() => Number, { nullable: true }) + FAILED_INSUFFICIENT_PERMISSIONS?: number; + + @Field(() => Number, { nullable: true }) + FAILED_UNKNOWN?: number; +} diff --git a/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts b/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts new file mode 100644 index 000000000..e8e122f2c --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts @@ -0,0 +1,15 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-queue-metrics.type'; + +@ObjectType() +export class WorkerQueueHealth { + @Field(() => String) + name: string; + + @Field(() => Number) + workers: number; + + @Field(() => WorkerQueueMetrics) + metrics: WorkerQueueMetrics; +} diff --git a/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-metrics.type.ts b/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-metrics.type.ts new file mode 100644 index 000000000..e24c35c9b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-metrics.type.ts @@ -0,0 +1,22 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +@ObjectType() +export class WorkerQueueMetrics { + @Field(() => Number) + failed: number; + + @Field(() => Number) + completed: number; + + @Field(() => Number) + waiting: number; + + @Field(() => Number) + active: number; + + @Field(() => Number) + delayed: number; + + @Field(() => Number) + prioritized: number; +} diff --git a/packages/twenty-server/src/engine/core-modules/health/utils/health-check-timeout.util.ts b/packages/twenty-server/src/engine/core-modules/health/utils/health-check-timeout.util.ts new file mode 100644 index 000000000..7504d6184 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/health/utils/health-check-timeout.util.ts @@ -0,0 +1,16 @@ +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; + +export const withHealthCheckTimeout = async ( + promise: Promise, + errorMessage: string, +): Promise => { + return Promise.race([ + promise, + new Promise((_, reject) => + setTimeout( + () => reject(new Error(errorMessage)), + HEALTH_INDICATORS_TIMEOUT, + ), + ), + ]); +}; diff --git a/yarn.lock b/yarn.lock index 27367dac3..d49f0bb1b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8421,26 +8421,27 @@ __metadata: languageName: node linkType: hard -"@nestjs/terminus@npm:^9.2.2": - version: 9.2.2 - resolution: "@nestjs/terminus@npm:9.2.2" +"@nestjs/terminus@npm:^11.0.0": + version: 11.0.0 + resolution: "@nestjs/terminus@npm:11.0.0" dependencies: boxen: "npm:5.1.2" - check-disk-space: "npm:3.3.1" + check-disk-space: "npm:3.4.0" peerDependencies: "@grpc/grpc-js": "*" "@grpc/proto-loader": "*" "@mikro-orm/core": "*" "@mikro-orm/nestjs": "*" - "@nestjs/axios": "*" - "@nestjs/common": 9.x - "@nestjs/core": 9.x - "@nestjs/microservices": "*" - "@nestjs/mongoose": "*" - "@nestjs/sequelize": "*" - "@nestjs/typeorm": "*" + "@nestjs/axios": ^2.0.0 || ^3.0.0 || ^4.0.0 + "@nestjs/common": ^10.0.0 || ^11.0.0 + "@nestjs/core": ^10.0.0 || ^11.0.0 + "@nestjs/microservices": ^10.0.0 || ^11.0.0 + "@nestjs/mongoose": ^11.0.0 + "@nestjs/sequelize": ^10.0.0 || ^11.0.0 + "@nestjs/typeorm": ^10.0.0 || ^11.0.0 + "@prisma/client": "*" mongoose: "*" - reflect-metadata: 0.1.x + reflect-metadata: 0.1.x || 0.2.x rxjs: 7.x sequelize: "*" typeorm: "*" @@ -8463,13 +8464,15 @@ __metadata: optional: true "@nestjs/typeorm": optional: true + "@prisma/client": + optional: true mongoose: optional: true sequelize: optional: true typeorm: optional: true - checksum: 10c0/3de4a3ce831c5a31ee2ce7327ffdb8180a1d4c270c18ff405484cd4bb60d225c001705add3df76b86221c3c7e1cc386ae60133b4cb4ac7f5e8dac62d2fd7cddb + checksum: 10c0/6c2d019b7a0f91bc68e654b44e45d93998d3a30919aaee8792e337d33183452817cd462706101f39157813d2aabb5d2ac81bc392c2ba943bfeaeb65920da15db languageName: node linkType: hard @@ -23510,10 +23513,10 @@ __metadata: languageName: node linkType: hard -"check-disk-space@npm:3.3.1": - version: 3.3.1 - resolution: "check-disk-space@npm:3.3.1" - checksum: 10c0/c5717bf4a2ae7b03bc6aff7a5c87149cbede3802f078ff56c8524d64329e788e2abcc270943ebc132819c40d110a0338c2670e962ce4f13d572da7a6c06af87c +"check-disk-space@npm:3.4.0": + version: 3.4.0 + resolution: "check-disk-space@npm:3.4.0" + checksum: 10c0/cc39c91e1337e974fb5069c2fbd9eb92aceca6e35f3da6863a4eada58f15c1bf6970055bffed1e41c15cde1fd0ad2580bb99bef8275791ed56d69947f8657aa5 languageName: node linkType: hard @@ -47133,7 +47136,7 @@ __metadata: "@nestjs/platform-express": "npm:^9.0.0" "@nestjs/schematics": "npm:^9.0.0" "@nestjs/serve-static": "npm:^4.0.1" - "@nestjs/terminus": "npm:^9.2.2" + "@nestjs/terminus": "npm:^11.0.0" "@nestjs/testing": "npm:^9.0.0" "@nestjs/typeorm": "npm:^10.0.0" "@next/eslint-plugin-next": "npm:^14.1.4"