Admin panel: App health check (#10546)
closes https://github.com/twentyhq/core-team-issues/issues/441 whats new - - app health with the proposed format -- update, scratched this format in favor of a more concise and more intentional health check. Now we will check app health only based on pendingMigrations <del> ``` status: system: { nodeVersion }, overview: { totalWorkspaces, criticalWorkspaces, workspacesWithPendingMigrations, healthDistribution: { healthy, warning, critical, } }, problematicWorkspaces: [ { workspaceId, severity, pendingMigrations, issuesSummary: { structural, data, relationship } } ] ``` </del> - errorMessage and details seperation -- before we used to send error in details which made it difficult if we want both error and details on the front -- usecase >> suppose app health indicator is not healthy but still want to send details - stateHistory with timestamp -- this is something I introduced, not sure about this. Basically the thought process was to store the LastState of the details, incase of no connection or timeout errors. This is not yet used on the front, just the endpoint. But it could be used on the front too - name unifying https://discord.com/channels/1130383047699738754/1346454192776155156 - json tree https://discord.com/channels/1130383047699738754/1346458558048501760 - match figma design https://discord.com/channels/1130383047699738754/1346451659647094815 - fix the collapse/open styles in tables https://discord.com/channels/1130383047699738754/1346452051974160406 - shift eye icon to expanded container https://discord.com/channels/1130383047699738754/1346452282669010987 - use H2Title for title and description of env variables groups https://discord.com/channels/1130383047699738754/1346434955936530452
This commit is contained in:
@ -30,6 +30,7 @@ export type AdminPanelHealthServiceData = {
|
||||
__typename?: 'AdminPanelHealthServiceData';
|
||||
description: Scalars['String'];
|
||||
details?: Maybe<Scalars['String']>;
|
||||
errorMessage?: Maybe<Scalars['String']>;
|
||||
id: Scalars['String'];
|
||||
label: Scalars['String'];
|
||||
queues?: Maybe<Array<AdminPanelWorkerQueueHealth>>;
|
||||
@ -642,6 +643,7 @@ export type GlobalSearchRecord = {
|
||||
};
|
||||
|
||||
export enum HealthIndicatorId {
|
||||
app = 'app',
|
||||
connectedAccount = 'connectedAccount',
|
||||
database = 'database',
|
||||
redis = 'redis',
|
||||
@ -2455,7 +2457,7 @@ export type GetIndicatorHealthStatusQueryVariables = Exact<{
|
||||
}>;
|
||||
|
||||
|
||||
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus }> | null } };
|
||||
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, errorMessage?: string | null, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus }> | null } };
|
||||
|
||||
export type GetQueueMetricsQueryVariables = Exact<{
|
||||
queueName: Scalars['String'];
|
||||
@ -4305,6 +4307,7 @@ export const GetIndicatorHealthStatusDocument = gql`
|
||||
label
|
||||
description
|
||||
status
|
||||
errorMessage
|
||||
details
|
||||
queues {
|
||||
id
|
||||
|
||||
@ -450,13 +450,13 @@ export const SettingsRoutes = ({
|
||||
|
||||
{isAdminPageEnabled && (
|
||||
<>
|
||||
<Route path={SettingsPath.AdminPanel} element={<SettingsAdmin />} />
|
||||
<Route path={SettingsPath.ServerAdmin} element={<SettingsAdmin />} />
|
||||
<Route
|
||||
path={SettingsPath.FeatureFlags}
|
||||
element={<SettingsAdminContent />}
|
||||
/>
|
||||
<Route
|
||||
path={SettingsPath.AdminPanelIndicatorHealthStatus}
|
||||
path={SettingsPath.ServerAdminIndicatorHealthStatus}
|
||||
element={<SettingsAdminIndicatorHealthStatus />}
|
||||
/>
|
||||
</>
|
||||
|
||||
@ -3,8 +3,8 @@ import { SETTINGS_ADMIN_TABS } from '@/settings/admin-panel/constants/SettingsAd
|
||||
import { SETTINGS_ADMIN_TABS_ID } from '@/settings/admin-panel/constants/SettingsAdminTabsId';
|
||||
import { TabList } from '@/ui/layout/tab/components/TabList';
|
||||
import styled from '@emotion/styled';
|
||||
import { IconHeart, IconSettings2, IconVariable } from 'twenty-ui';
|
||||
import { t } from '@lingui/core/macro';
|
||||
import { IconHeart, IconSettings2, IconVariable } from 'twenty-ui';
|
||||
|
||||
const StyledTabListContainer = styled.div`
|
||||
align-items: center;
|
||||
|
||||
@ -2,22 +2,13 @@ import { SettingsAdminEnvVariablesTable } from '@/settings/admin-panel/component
|
||||
import { SettingsAdminTabSkeletonLoader } from '@/settings/admin-panel/components/SettingsAdminTabSkeletonLoader';
|
||||
import styled from '@emotion/styled';
|
||||
import { useState } from 'react';
|
||||
import { Button, H1Title, H1TitleFontColor, Section } from 'twenty-ui';
|
||||
import { Button, H1Title, H1TitleFontColor, H2Title, Section } from 'twenty-ui';
|
||||
import { useGetEnvironmentVariablesGroupedQuery } from '~/generated/graphql';
|
||||
|
||||
const StyledGroupContainer = styled.div`
|
||||
margin-bottom: ${({ theme }) => theme.spacing(6)};
|
||||
`;
|
||||
|
||||
const StyledGroupVariablesContainer = styled.div`
|
||||
background-color: ${({ theme }) => theme.background.secondary};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||
padding-bottom: ${({ theme }) => theme.spacing(2)};
|
||||
padding-left: ${({ theme }) => theme.spacing(4)};
|
||||
padding-right: ${({ theme }) => theme.spacing(4)};
|
||||
`;
|
||||
|
||||
const StyledGroupDescription = styled.div`
|
||||
margin-bottom: ${({ theme }) => theme.spacing(4)};
|
||||
`;
|
||||
@ -78,16 +69,9 @@ export const SettingsAdminEnvVariables = () => {
|
||||
<Section>
|
||||
{visibleGroups.map((group) => (
|
||||
<StyledGroupContainer key={group.name}>
|
||||
<H1Title title={group.name} fontColor={H1TitleFontColor.Primary} />
|
||||
{group.description !== '' && (
|
||||
<StyledGroupDescription>
|
||||
{group.description}
|
||||
</StyledGroupDescription>
|
||||
)}
|
||||
<H2Title title={group.name} description={group.description} />
|
||||
{group.variables.length > 0 && (
|
||||
<StyledGroupVariablesContainer>
|
||||
<SettingsAdminEnvVariablesTable variables={group.variables} />
|
||||
</StyledGroupVariablesContainer>
|
||||
<SettingsAdminEnvVariablesTable variables={group.variables} />
|
||||
)}
|
||||
</StyledGroupContainer>
|
||||
))}
|
||||
@ -120,11 +104,9 @@ export const SettingsAdminEnvVariables = () => {
|
||||
</StyledGroupDescription>
|
||||
)}
|
||||
{selectedGroupData.variables.length > 0 && (
|
||||
<StyledGroupVariablesContainer>
|
||||
<SettingsAdminEnvVariablesTable
|
||||
variables={selectedGroupData.variables}
|
||||
/>
|
||||
</StyledGroupVariablesContainer>
|
||||
<SettingsAdminEnvVariablesTable
|
||||
variables={selectedGroupData.variables}
|
||||
/>
|
||||
)}
|
||||
</StyledGroupContainer>
|
||||
)}
|
||||
|
||||
@ -2,6 +2,7 @@ import { TableCell } from '@/ui/layout/table/components/TableCell';
|
||||
import { TableRow } from '@/ui/layout/table/components/TableRow';
|
||||
import { useTheme } from '@emotion/react';
|
||||
import styled from '@emotion/styled';
|
||||
import { motion } from 'framer-motion';
|
||||
import { useState } from 'react';
|
||||
import {
|
||||
AnimatedExpandableContainer,
|
||||
@ -10,6 +11,7 @@ import {
|
||||
IconEyeOff,
|
||||
LightIconButton,
|
||||
} from 'twenty-ui';
|
||||
|
||||
type SettingsAdminEnvVariablesRowProps = {
|
||||
variable: {
|
||||
name: string;
|
||||
@ -26,8 +28,23 @@ const StyledTruncatedCell = styled(TableCell)`
|
||||
cursor: pointer;
|
||||
`;
|
||||
|
||||
const StyledButton = styled(motion.button)`
|
||||
align-items: center;
|
||||
border: none;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding-inline: ${({ theme }) => theme.spacing(1)};
|
||||
background-color: transparent;
|
||||
height: 24px;
|
||||
width: 24px;
|
||||
box-sizing: border-box;
|
||||
cursor: pointer;
|
||||
`;
|
||||
|
||||
const MotionIconChevronDown = motion(IconChevronRight);
|
||||
|
||||
const StyledExpandedDetails = styled.div`
|
||||
background-color: ${({ theme }) => theme.background.tertiary};
|
||||
background-color: ${({ theme }) => theme.background.secondary};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
margin: ${({ theme }) => theme.spacing(2)} 0;
|
||||
padding: ${({ theme }) => theme.spacing(2)};
|
||||
@ -40,7 +57,8 @@ const StyledExpandedDetails = styled.div`
|
||||
`;
|
||||
|
||||
const StyledDetailLabel = styled.div`
|
||||
font-weight: ${({ theme }) => theme.font.weight.medium};
|
||||
color: ${({ theme }) => theme.font.color.tertiary};
|
||||
font-weight: ${({ theme }) => theme.font.weight.regular};
|
||||
padding-right: ${({ theme }) => theme.spacing(4)};
|
||||
`;
|
||||
|
||||
@ -56,12 +74,16 @@ const StyledExpandedLabel = styled.div`
|
||||
overflow: visible;
|
||||
`;
|
||||
|
||||
const StyledTransitionedIconChevronRight = styled(IconChevronRight)`
|
||||
cursor: pointer;
|
||||
transform: ${({ $isExpanded }: { $isExpanded: boolean }) =>
|
||||
$isExpanded ? 'rotate(90deg)' : 'rotate(0deg)'};
|
||||
transition: ${({ theme }) =>
|
||||
`transform ${theme.animation.duration.normal}s ease`};
|
||||
const StyledValueContainer = styled.div`
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
`;
|
||||
|
||||
const StyledTableRow = styled(TableRow)<{ isExpanded: boolean }>`
|
||||
background-color: ${({ isExpanded, theme }) =>
|
||||
isExpanded ? theme.background.transparent.light : 'transparent'};
|
||||
margin-bottom: ${({ theme }) => theme.spacing(0.5)};
|
||||
`;
|
||||
|
||||
export const SettingsAdminEnvVariablesRow = ({
|
||||
@ -85,44 +107,51 @@ export const SettingsAdminEnvVariablesRow = ({
|
||||
|
||||
return (
|
||||
<>
|
||||
<TableRow
|
||||
<StyledTableRow
|
||||
onClick={() => setIsExpanded(!isExpanded)}
|
||||
gridAutoColumns="4fr 3fr 2fr 1fr 1fr"
|
||||
gridAutoColumns="5fr 4fr 3fr 1fr"
|
||||
isExpanded={isExpanded}
|
||||
>
|
||||
<StyledTruncatedCell color="primary">
|
||||
<StyledTruncatedCell color={theme.font.color.primary}>
|
||||
<StyledEllipsisLabel>{variable.name}</StyledEllipsisLabel>
|
||||
</StyledTruncatedCell>
|
||||
<StyledTruncatedCell>
|
||||
<StyledEllipsisLabel>{variable.description}</StyledEllipsisLabel>
|
||||
</StyledTruncatedCell>
|
||||
<StyledTruncatedCell>
|
||||
<StyledTruncatedCell align="right">
|
||||
<StyledEllipsisLabel>{displayValue}</StyledEllipsisLabel>
|
||||
</StyledTruncatedCell>
|
||||
<TableCell align="right">
|
||||
{variable.sensitive && variable.value !== '' && (
|
||||
<LightIconButton
|
||||
Icon={showSensitiveValue ? IconEyeOff : IconEye}
|
||||
size="small"
|
||||
accent="secondary"
|
||||
onClick={handleToggleVisibility}
|
||||
<StyledButton onClick={() => setIsExpanded(!isExpanded)}>
|
||||
<MotionIconChevronDown
|
||||
size={theme.icon.size.md}
|
||||
color={theme.font.color.tertiary}
|
||||
initial={false}
|
||||
animate={{ rotate: isExpanded ? 90 : 0 }}
|
||||
/>
|
||||
)}
|
||||
</StyledButton>
|
||||
</TableCell>
|
||||
<TableCell align="right">
|
||||
<StyledTransitionedIconChevronRight
|
||||
$isExpanded={isExpanded}
|
||||
size={theme.icon.size.sm}
|
||||
/>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</StyledTableRow>
|
||||
<AnimatedExpandableContainer isExpanded={isExpanded} mode="fit-content">
|
||||
<StyledExpandedDetails>
|
||||
<StyledDetailLabel>Name:</StyledDetailLabel>
|
||||
<StyledDetailLabel>Name</StyledDetailLabel>
|
||||
<StyledEllipsisLabel>{variable.name}</StyledEllipsisLabel>
|
||||
<StyledDetailLabel>Description:</StyledDetailLabel>
|
||||
<StyledDetailLabel>Description</StyledDetailLabel>
|
||||
<StyledExpandedLabel>{variable.description}</StyledExpandedLabel>
|
||||
<StyledDetailLabel>Value:</StyledDetailLabel>
|
||||
<StyledExpandedLabel>{displayValue}</StyledExpandedLabel>
|
||||
<StyledDetailLabel>Value</StyledDetailLabel>
|
||||
<StyledExpandedLabel>
|
||||
<StyledValueContainer>
|
||||
{displayValue}
|
||||
{variable.sensitive && variable.value !== '' && (
|
||||
<LightIconButton
|
||||
Icon={showSensitiveValue ? IconEyeOff : IconEye}
|
||||
size="small"
|
||||
accent="secondary"
|
||||
onClick={handleToggleVisibility}
|
||||
/>
|
||||
)}
|
||||
</StyledValueContainer>
|
||||
</StyledExpandedLabel>
|
||||
</StyledExpandedDetails>
|
||||
</AnimatedExpandableContainer>
|
||||
</>
|
||||
|
||||
@ -21,11 +21,10 @@ export const SettingsAdminEnvVariablesTable = ({
|
||||
variables,
|
||||
}: SettingsAdminEnvVariablesTableProps) => (
|
||||
<StyledTable>
|
||||
<TableRow gridAutoColumns="4fr 3fr 2fr 1fr 1fr">
|
||||
<TableRow gridAutoColumns="5fr 4fr 3fr 1fr">
|
||||
<TableHeader>Name</TableHeader>
|
||||
<TableHeader>Description</TableHeader>
|
||||
<TableHeader>Value</TableHeader>
|
||||
<TableHeader align="right"></TableHeader>
|
||||
<TableHeader align="right">Value</TableHeader>
|
||||
<TableHeader align="right"></TableHeader>
|
||||
</TableRow>
|
||||
{variables.map((variable) => (
|
||||
|
||||
@ -17,11 +17,13 @@ export const ConnectedAccountHealthStatus = () => {
|
||||
}
|
||||
|
||||
const parsedDetails = JSON.parse(details);
|
||||
const serviceDetails = parsedDetails.details;
|
||||
|
||||
const isMessageSyncDown =
|
||||
parsedDetails.messageSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
|
||||
serviceDetails.messageSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
|
||||
const isCalendarSyncDown =
|
||||
parsedDetails.calendarSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
|
||||
serviceDetails.calendarSync?.status ===
|
||||
AdminPanelHealthServiceStatus.OUTAGE;
|
||||
|
||||
const errorMessages = [];
|
||||
if (isMessageSyncDown) {
|
||||
@ -39,16 +41,16 @@ export const ConnectedAccountHealthStatus = () => {
|
||||
</StyledErrorMessage>
|
||||
)}
|
||||
|
||||
{!isMessageSyncDown && parsedDetails.messageSync?.details && (
|
||||
{!isMessageSyncDown && serviceDetails.messageSync?.details && (
|
||||
<SettingsAdminHealthAccountSyncCountersTable
|
||||
details={parsedDetails.messageSync.details}
|
||||
details={serviceDetails.messageSync.details}
|
||||
title="Message Sync Status"
|
||||
/>
|
||||
)}
|
||||
|
||||
{!isCalendarSyncDown && parsedDetails.calendarSync?.details && (
|
||||
{!isCalendarSyncDown && serviceDetails.calendarSync?.details && (
|
||||
<SettingsAdminHealthAccountSyncCountersTable
|
||||
details={parsedDetails.calendarSync.details}
|
||||
details={serviceDetails.calendarSync.details}
|
||||
title="Calendar Sync Status"
|
||||
/>
|
||||
)}
|
||||
|
||||
@ -1,43 +0,0 @@
|
||||
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
|
||||
import styled from '@emotion/styled';
|
||||
import { useContext } from 'react';
|
||||
import { Section } from 'twenty-ui';
|
||||
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
|
||||
|
||||
const StyledDetailsContainer = styled.pre`
|
||||
background-color: ${({ theme }) => theme.background.quaternary};
|
||||
padding: ${({ theme }) => theme.spacing(6)};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
white-space: pre-wrap;
|
||||
font-size: ${({ theme }) => theme.font.size.sm};
|
||||
margin: 0;
|
||||
`;
|
||||
|
||||
const StyledErrorMessage = styled.div`
|
||||
color: ${({ theme }) => theme.color.red};
|
||||
margin-top: ${({ theme }) => theme.spacing(2)};
|
||||
`;
|
||||
|
||||
export const DatabaseAndRedisHealthStatus = () => {
|
||||
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
|
||||
|
||||
const formattedDetails = indicatorHealth.details
|
||||
? JSON.stringify(JSON.parse(indicatorHealth.details), null, 2)
|
||||
: null;
|
||||
|
||||
const isDatabaseOrRedisDown =
|
||||
!indicatorHealth.status ||
|
||||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
|
||||
|
||||
return (
|
||||
<Section>
|
||||
{isDatabaseOrRedisDown ? (
|
||||
<StyledErrorMessage>
|
||||
{`${indicatorHealth.label} information is not available because the service is down`}
|
||||
</StyledErrorMessage>
|
||||
) : (
|
||||
<StyledDetailsContainer>{formattedDetails}</StyledDetailsContainer>
|
||||
)}
|
||||
</Section>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,49 @@
|
||||
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
|
||||
import { JsonTree } from '@/workflow/components/json-visualizer/components/JsonTree';
|
||||
import styled from '@emotion/styled';
|
||||
import { useContext } from 'react';
|
||||
import { Section } from 'twenty-ui';
|
||||
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
|
||||
|
||||
const StyledDetailsContainer = styled.div`
|
||||
background-color: ${({ theme }) => theme.background.secondary};
|
||||
padding: ${({ theme }) => theme.spacing(4)};
|
||||
border-radius: ${({ theme }) => theme.border.radius.md};
|
||||
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||
white-space: pre-wrap;
|
||||
font-size: ${({ theme }) => theme.font.size.sm};
|
||||
`;
|
||||
|
||||
const StyledErrorMessage = styled.div`
|
||||
color: ${({ theme }) => theme.color.red};
|
||||
margin-top: ${({ theme }) => theme.spacing(2)};
|
||||
margin-bottom: ${({ theme }) => theme.spacing(4)};
|
||||
`;
|
||||
|
||||
export const JsonDataIndicatorHealthStatus = () => {
|
||||
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
|
||||
|
||||
const parsedDetails = indicatorHealth.details
|
||||
? JSON.parse(indicatorHealth.details)
|
||||
: null;
|
||||
|
||||
const isDown =
|
||||
!indicatorHealth.status ||
|
||||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
|
||||
|
||||
return (
|
||||
<Section>
|
||||
{isDown && (
|
||||
<StyledErrorMessage>
|
||||
{indicatorHealth.errorMessage ||
|
||||
`${indicatorHealth.label} service is unreachable`}
|
||||
</StyledErrorMessage>
|
||||
)}
|
||||
{parsedDetails && (
|
||||
<StyledDetailsContainer>
|
||||
<JsonTree value={parsedDetails} />
|
||||
</StyledDetailsContainer>
|
||||
)}
|
||||
</Section>
|
||||
);
|
||||
};
|
||||
@ -1,5 +1,5 @@
|
||||
import { ConnectedAccountHealthStatus } from '@/settings/admin-panel/health-status/components/ConnectedAccountHealthStatus';
|
||||
import { DatabaseAndRedisHealthStatus } from '@/settings/admin-panel/health-status/components/DatabaseAndRedisHealthStatus';
|
||||
import { JsonDataIndicatorHealthStatus } from '@/settings/admin-panel/health-status/components/JsonDataIndicatorHealthStatus';
|
||||
import { WorkerHealthStatus } from '@/settings/admin-panel/health-status/components/WorkerHealthStatus';
|
||||
import { useParams } from 'react-router-dom';
|
||||
import { HealthIndicatorId } from '~/generated/graphql';
|
||||
@ -10,11 +10,13 @@ export const SettingsAdminIndicatorHealthStatusContent = () => {
|
||||
switch (indicatorId) {
|
||||
case HealthIndicatorId.database:
|
||||
case HealthIndicatorId.redis:
|
||||
return <DatabaseAndRedisHealthStatus />;
|
||||
case HealthIndicatorId.app:
|
||||
return <JsonDataIndicatorHealthStatus />;
|
||||
case HealthIndicatorId.worker:
|
||||
return <WorkerHealthStatus />;
|
||||
case HealthIndicatorId.connectedAccount:
|
||||
return <ConnectedAccountHealthStatus />;
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -20,7 +20,7 @@ export const SettingsHealthStatusListCard = ({
|
||||
<SettingsAdminHealthStatusRightContainer status={service.status} />
|
||||
)}
|
||||
to={(service) =>
|
||||
getSettingsPath(SettingsPath.AdminPanelIndicatorHealthStatus, {
|
||||
getSettingsPath(SettingsPath.ServerAdminIndicatorHealthStatus, {
|
||||
indicatorId: service.id,
|
||||
})
|
||||
}
|
||||
|
||||
@ -14,6 +14,7 @@ export const SettingsAdminIndicatorHealthContext =
|
||||
id: '',
|
||||
label: '',
|
||||
description: '',
|
||||
errorMessage: '',
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: '',
|
||||
queues: [],
|
||||
|
||||
@ -7,6 +7,7 @@ export const GET_INDICATOR_HEALTH_STATUS = gql`
|
||||
label
|
||||
description
|
||||
status
|
||||
errorMessage
|
||||
details
|
||||
queues {
|
||||
id
|
||||
|
||||
@ -179,7 +179,7 @@ const useSettingsNavigationItems = (): SettingsNavigationSection[] => {
|
||||
items: [
|
||||
{
|
||||
label: t`Server Admin`,
|
||||
path: SettingsPath.AdminPanel,
|
||||
path: SettingsPath.ServerAdmin,
|
||||
Icon: IconServer,
|
||||
isHidden: !isAdminEnabled,
|
||||
},
|
||||
|
||||
@ -34,10 +34,10 @@ export enum SettingsPath {
|
||||
DevelopersNewWebhook = 'developers/webhooks/new',
|
||||
DevelopersNewWebhookDetail = 'developers/webhooks/:webhookId',
|
||||
Releases = 'releases',
|
||||
AdminPanel = 'admin-panel',
|
||||
FeatureFlags = 'admin-panel/feature-flags',
|
||||
AdminPanelHealthStatus = 'admin-panel#health-status',
|
||||
AdminPanelIndicatorHealthStatus = 'admin-panel/health-status/:indicatorId',
|
||||
ServerAdmin = 'server-admin',
|
||||
FeatureFlags = 'server-admin/feature-flags',
|
||||
ServerAdminHealthStatus = 'server-admin#health-status',
|
||||
ServerAdminIndicatorHealthStatus = 'server-admin/health-status/:indicatorId',
|
||||
Lab = 'lab',
|
||||
Roles = 'roles',
|
||||
RoleDetail = 'roles/:roleId',
|
||||
|
||||
@ -14,9 +14,9 @@ export const SettingsAdmin = () => {
|
||||
links={[
|
||||
{
|
||||
children: t`Other`,
|
||||
href: getSettingsPath(SettingsPath.AdminPanel),
|
||||
href: getSettingsPath(SettingsPath.ServerAdmin),
|
||||
},
|
||||
{ children: t`Server Admin Panel` },
|
||||
{ children: t`Server Admin` },
|
||||
]}
|
||||
>
|
||||
<SettingsPageContainer>
|
||||
|
||||
@ -20,6 +20,17 @@ const StyledH2Title = styled(H2Title)`
|
||||
margin-top: ${({ theme }) => theme.spacing(2)};
|
||||
`;
|
||||
|
||||
const StyledTitleContainer = styled.div`
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: ${({ theme }) => theme.spacing(2)};
|
||||
`;
|
||||
|
||||
const StyledHealthStatusContainer = styled.div`
|
||||
margin-bottom: ${({ theme }) => theme.spacing(4)};
|
||||
margin-top: ${({ theme }) => theme.spacing(1)};
|
||||
`;
|
||||
|
||||
export const SettingsAdminIndicatorHealthStatus = () => {
|
||||
const { t } = useLingui();
|
||||
const { indicatorId } = useParams();
|
||||
@ -40,15 +51,15 @@ export const SettingsAdminIndicatorHealthStatus = () => {
|
||||
links={[
|
||||
{
|
||||
children: t`Other`,
|
||||
href: getSettingsPath(SettingsPath.AdminPanel),
|
||||
href: getSettingsPath(SettingsPath.ServerAdmin),
|
||||
},
|
||||
{
|
||||
children: t`Server Admin Panel`,
|
||||
href: getSettingsPath(SettingsPath.AdminPanel),
|
||||
children: t`Server Admin`,
|
||||
href: getSettingsPath(SettingsPath.ServerAdmin),
|
||||
},
|
||||
{
|
||||
children: t`Health Status`,
|
||||
href: getSettingsPath(SettingsPath.AdminPanelHealthStatus),
|
||||
href: getSettingsPath(SettingsPath.ServerAdminHealthStatus),
|
||||
},
|
||||
{ children: `${data?.getIndicatorHealthStatus?.label}` },
|
||||
]}
|
||||
@ -60,6 +71,7 @@ export const SettingsAdminIndicatorHealthStatus = () => {
|
||||
id: data?.getIndicatorHealthStatus?.id ?? '',
|
||||
label: data?.getIndicatorHealthStatus?.label ?? '',
|
||||
description: data?.getIndicatorHealthStatus?.description ?? '',
|
||||
errorMessage: data?.getIndicatorHealthStatus?.errorMessage,
|
||||
status:
|
||||
data?.getIndicatorHealthStatus?.status ??
|
||||
AdminPanelHealthServiceStatus.OUTAGE,
|
||||
@ -69,16 +81,20 @@ export const SettingsAdminIndicatorHealthStatus = () => {
|
||||
}}
|
||||
>
|
||||
<Section>
|
||||
<StyledH2Title
|
||||
title={`${data?.getIndicatorHealthStatus?.label}`}
|
||||
description={data?.getIndicatorHealthStatus?.description}
|
||||
/>
|
||||
{indicatorId !== HealthIndicatorId.connectedAccount &&
|
||||
data?.getIndicatorHealthStatus?.status && (
|
||||
<SettingsAdminHealthStatusRightContainer
|
||||
status={data?.getIndicatorHealthStatus.status}
|
||||
/>
|
||||
)}
|
||||
<StyledTitleContainer>
|
||||
<StyledH2Title
|
||||
title={`${data?.getIndicatorHealthStatus?.label}`}
|
||||
description={data?.getIndicatorHealthStatus?.description}
|
||||
/>
|
||||
{indicatorId !== HealthIndicatorId.connectedAccount &&
|
||||
data?.getIndicatorHealthStatus?.status && (
|
||||
<StyledHealthStatusContainer>
|
||||
<SettingsAdminHealthStatusRightContainer
|
||||
status={data?.getIndicatorHealthStatus.status}
|
||||
/>
|
||||
</StyledHealthStatusContainer>
|
||||
)}
|
||||
</StyledTitleContainer>
|
||||
</Section>
|
||||
|
||||
<SettingsAdminIndicatorHealthStatusContent />
|
||||
|
||||
@ -11,6 +11,7 @@ import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums
|
||||
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
@ -26,6 +27,7 @@ describe('AdminPanelHealthService', () => {
|
||||
let redisHealth: jest.Mocked<RedisHealthIndicator>;
|
||||
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
|
||||
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
|
||||
let appHealth: jest.Mocked<AppHealthIndicator>;
|
||||
let redisClient: jest.Mocked<RedisClientService>;
|
||||
let environmentService: jest.Mocked<EnvironmentService>;
|
||||
let loggerSpy: jest.SpyInstance;
|
||||
@ -35,6 +37,7 @@ describe('AdminPanelHealthService', () => {
|
||||
redisHealth = { isHealthy: jest.fn() } as any;
|
||||
workerHealth = { isHealthy: jest.fn(), getQueueDetails: jest.fn() } as any;
|
||||
connectedAccountHealth = { isHealthy: jest.fn() } as any;
|
||||
appHealth = { isHealthy: jest.fn() } as any;
|
||||
redisClient = {
|
||||
getClient: jest.fn().mockReturnValue({} as Redis),
|
||||
} as any;
|
||||
@ -53,6 +56,7 @@ describe('AdminPanelHealthService', () => {
|
||||
{ provide: RedisHealthIndicator, useValue: redisHealth },
|
||||
{ provide: WorkerHealthIndicator, useValue: workerHealth },
|
||||
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
|
||||
{ provide: AppHealthIndicator, useValue: appHealth },
|
||||
{ provide: RedisClientService, useValue: redisClient },
|
||||
{ provide: EnvironmentService, useValue: environmentService },
|
||||
],
|
||||
@ -108,6 +112,32 @@ describe('AdminPanelHealthService', () => {
|
||||
details: 'Account sync is operational',
|
||||
},
|
||||
});
|
||||
appHealth.isHealthy.mockResolvedValue({
|
||||
app: {
|
||||
status: 'up',
|
||||
details: {
|
||||
system: {
|
||||
nodeVersion: '16.0',
|
||||
},
|
||||
workspaces: {
|
||||
totalWorkspaces: 1,
|
||||
healthStatus: [
|
||||
{
|
||||
workspaceId: '1',
|
||||
summary: {
|
||||
structuralIssues: 0,
|
||||
dataIssues: 0,
|
||||
relationshipIssues: 0,
|
||||
pendingMigrations: 0,
|
||||
},
|
||||
severity: 'healthy',
|
||||
details: {},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
|
||||
@ -129,6 +159,10 @@ describe('AdminPanelHealthService', () => {
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.app],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
@ -148,6 +182,9 @@ describe('AdminPanelHealthService', () => {
|
||||
connectedAccountHealth.isHealthy.mockResolvedValue({
|
||||
connectedAccount: { status: 'up' },
|
||||
});
|
||||
appHealth.isHealthy.mockResolvedValue({
|
||||
app: { status: 'down', details: {} },
|
||||
});
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
|
||||
@ -169,6 +206,10 @@ describe('AdminPanelHealthService', () => {
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.app],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
@ -186,6 +227,9 @@ describe('AdminPanelHealthService', () => {
|
||||
connectedAccountHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED),
|
||||
);
|
||||
appHealth.isHealthy.mockRejectedValue(
|
||||
new Error(HEALTH_ERROR_MESSAGES.APP_HEALTH_CHECK_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.getSystemHealthStatus();
|
||||
|
||||
@ -207,6 +251,10 @@ describe('AdminPanelHealthService', () => {
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.app],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
@ -233,7 +281,8 @@ describe('AdminPanelHealthService', () => {
|
||||
expect(result).toStrictEqual({
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.database],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: JSON.stringify(details),
|
||||
details: JSON.stringify({ details }),
|
||||
errorMessage: undefined,
|
||||
queues: undefined,
|
||||
});
|
||||
});
|
||||
@ -266,7 +315,8 @@ describe('AdminPanelHealthService', () => {
|
||||
expect(result).toStrictEqual({
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.worker],
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: undefined,
|
||||
details: JSON.stringify({ queues: mockQueues }),
|
||||
errorMessage: undefined,
|
||||
queues: mockQueues.map((queue) => ({
|
||||
id: `worker-${queue.queueName}`,
|
||||
queueName: queue.queueName,
|
||||
@ -290,7 +340,8 @@ describe('AdminPanelHealthService', () => {
|
||||
expect(result).toStrictEqual({
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.redis],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
details: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
|
||||
details: undefined,
|
||||
errorMessage: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@ import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-he
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
@ -27,6 +28,7 @@ export class AdminPanelHealthService {
|
||||
private readonly redisHealth: RedisHealthIndicator,
|
||||
private readonly workerHealth: WorkerHealthIndicator,
|
||||
private readonly connectedAccountHealth: ConnectedAccountHealth,
|
||||
private readonly appHealth: AppHealthIndicator,
|
||||
private readonly redisClient: RedisClientService,
|
||||
) {}
|
||||
|
||||
@ -35,6 +37,7 @@ export class AdminPanelHealthService {
|
||||
[HealthIndicatorId.redis]: this.redisHealth,
|
||||
[HealthIndicatorId.worker]: this.workerHealth,
|
||||
[HealthIndicatorId.connectedAccount]: this.connectedAccountHealth,
|
||||
[HealthIndicatorId.app]: this.appHealth,
|
||||
};
|
||||
|
||||
private transformStatus(status: HealthIndicatorStatus) {
|
||||
@ -46,15 +49,17 @@ export class AdminPanelHealthService {
|
||||
private transformServiceDetails(details: any) {
|
||||
if (!details) return details;
|
||||
|
||||
if (details.messageSync) {
|
||||
details.messageSync.status = this.transformStatus(
|
||||
details.messageSync.status,
|
||||
);
|
||||
}
|
||||
if (details.calendarSync) {
|
||||
details.calendarSync.status = this.transformStatus(
|
||||
details.calendarSync.status,
|
||||
);
|
||||
if (details.details) {
|
||||
if (details.details.messageSync) {
|
||||
details.details.messageSync.status = this.transformStatus(
|
||||
details.details.messageSync.status,
|
||||
);
|
||||
}
|
||||
if (details.details.calendarSync) {
|
||||
details.details.calendarSync.status = this.transformStatus(
|
||||
details.details.calendarSync.status,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return details;
|
||||
@ -65,17 +70,33 @@ export class AdminPanelHealthService {
|
||||
indicatorId: HealthIndicatorId,
|
||||
) {
|
||||
if (result.status === 'fulfilled') {
|
||||
const key = Object.keys(result.value)[0];
|
||||
const keys = Object.keys(result.value);
|
||||
|
||||
if (keys.length === 0) {
|
||||
return {
|
||||
...HEALTH_INDICATORS[indicatorId],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
errorMessage: 'No health check result available',
|
||||
};
|
||||
}
|
||||
const key = keys[0];
|
||||
const serviceResult = result.value[key];
|
||||
const details = this.transformServiceDetails(serviceResult.details);
|
||||
const { status, message, ...detailsWithoutStatus } = serviceResult;
|
||||
const indicator = HEALTH_INDICATORS[indicatorId];
|
||||
|
||||
const transformedDetails =
|
||||
this.transformServiceDetails(detailsWithoutStatus);
|
||||
|
||||
return {
|
||||
id: indicatorId,
|
||||
label: indicator.label,
|
||||
description: indicator.description,
|
||||
status: this.transformStatus(serviceResult.status),
|
||||
details: details ? JSON.stringify(details) : undefined,
|
||||
status: this.transformStatus(status),
|
||||
errorMessage: message,
|
||||
details:
|
||||
Object.keys(transformedDetails).length > 0
|
||||
? JSON.stringify(transformedDetails)
|
||||
: undefined,
|
||||
queues: serviceResult.queues,
|
||||
};
|
||||
}
|
||||
@ -83,7 +104,10 @@ export class AdminPanelHealthService {
|
||||
return {
|
||||
...HEALTH_INDICATORS[indicatorId],
|
||||
status: AdminPanelHealthServiceStatus.OUTAGE,
|
||||
details: result.reason?.message?.toString(),
|
||||
errorMessage: result.reason?.message?.toString(),
|
||||
details: result.reason?.details
|
||||
? JSON.stringify(result.reason.details)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@ -117,13 +141,19 @@ export class AdminPanelHealthService {
|
||||
}
|
||||
|
||||
async getSystemHealthStatus(): Promise<SystemHealth> {
|
||||
const [databaseResult, redisResult, workerResult, accountSyncResult] =
|
||||
await Promise.allSettled([
|
||||
this.databaseHealth.isHealthy(),
|
||||
this.redisHealth.isHealthy(),
|
||||
this.workerHealth.isHealthy(),
|
||||
this.connectedAccountHealth.isHealthy(),
|
||||
]);
|
||||
const [
|
||||
databaseResult,
|
||||
redisResult,
|
||||
workerResult,
|
||||
accountSyncResult,
|
||||
appResult,
|
||||
] = await Promise.allSettled([
|
||||
this.databaseHealth.isHealthy(),
|
||||
this.redisHealth.isHealthy(),
|
||||
this.workerHealth.isHealthy(),
|
||||
this.connectedAccountHealth.isHealthy(),
|
||||
this.appHealth.isHealthy(),
|
||||
]);
|
||||
|
||||
return {
|
||||
services: [
|
||||
@ -151,6 +181,11 @@ export class AdminPanelHealthService {
|
||||
HealthIndicatorId.connectedAccount,
|
||||
).status,
|
||||
},
|
||||
{
|
||||
...HEALTH_INDICATORS[HealthIndicatorId.app],
|
||||
status: this.getServiceStatus(appResult, HealthIndicatorId.app)
|
||||
.status,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
@ -28,4 +28,9 @@ export const HEALTH_INDICATORS: Record<HealthIndicatorId, HealthIndicatorInfo> =
|
||||
label: 'Connected Account Status',
|
||||
description: 'Connected accounts status',
|
||||
},
|
||||
[HealthIndicatorId.app]: {
|
||||
id: HealthIndicatorId.app,
|
||||
label: 'App Status',
|
||||
description: 'Workspace metadata migration status check',
|
||||
},
|
||||
};
|
||||
|
||||
@ -17,6 +17,8 @@ export class AdminPanelHealthServiceData {
|
||||
@Field(() => AdminPanelHealthServiceStatus)
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
errorMessage?: string;
|
||||
@Field(() => String, { nullable: true })
|
||||
details?: string;
|
||||
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
import { registerEnumType } from '@nestjs/graphql';
|
||||
|
||||
export enum AdminPanelHealthServiceStatus {
|
||||
OPERATIONAL = 'operational',
|
||||
OUTAGE = 'outage',
|
||||
OPERATIONAL = 'OPERATIONAL',
|
||||
OUTAGE = 'OUTAGE',
|
||||
}
|
||||
|
||||
registerEnumType(AdminPanelHealthServiceStatus, {
|
||||
|
||||
@ -12,4 +12,5 @@ export const HEALTH_ERROR_MESSAGES = {
|
||||
CALENDAR_SYNC_TIMEOUT: 'Calendar sync check timeout',
|
||||
CALENDAR_SYNC_CHECK_FAILED: 'Calendar sync check failed',
|
||||
CALENDAR_SYNC_HIGH_FAILURE_RATE: 'High failure rate in calendar sync jobs',
|
||||
APP_HEALTH_CHECK_FAILED: 'App health check failed',
|
||||
} as const;
|
||||
|
||||
@ -2,6 +2,7 @@ import { HealthCheckService } from '@nestjs/terminus';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
@ -34,6 +35,10 @@ describe('HealthController', () => {
|
||||
provide: ConnectedAccountHealth,
|
||||
useValue: { isHealthy: jest.fn() },
|
||||
},
|
||||
{
|
||||
provide: AppHealthIndicator,
|
||||
useValue: { isHealthy: jest.fn() },
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
|
||||
@ -2,11 +2,11 @@ import { BadRequestException, Controller, Get, Param } from '@nestjs/common';
|
||||
import { HealthCheck, HealthCheckService } from '@nestjs/terminus';
|
||||
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
|
||||
@Controller('healthz')
|
||||
export class HealthController {
|
||||
constructor(
|
||||
@ -15,6 +15,7 @@ export class HealthController {
|
||||
private readonly redisHealth: RedisHealthIndicator,
|
||||
private readonly workerHealth: WorkerHealthIndicator,
|
||||
private readonly connectedAccountHealth: ConnectedAccountHealth,
|
||||
private readonly appHealth: AppHealthIndicator,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
@ -32,6 +33,7 @@ export class HealthController {
|
||||
[HealthIndicatorId.worker]: () => this.workerHealth.isHealthy(),
|
||||
[HealthIndicatorId.connectedAccount]: () =>
|
||||
this.connectedAccountHealth.isHealthy(),
|
||||
[HealthIndicatorId.app]: () => this.appHealth.isHealthy(),
|
||||
};
|
||||
|
||||
if (!(indicatorId in checks)) {
|
||||
|
||||
@ -5,6 +5,7 @@ export enum HealthIndicatorId {
|
||||
redis = 'redis',
|
||||
worker = 'worker',
|
||||
connectedAccount = 'connectedAccount',
|
||||
app = 'app',
|
||||
}
|
||||
|
||||
registerEnumType(HealthIndicatorId, {
|
||||
|
||||
@ -3,7 +3,11 @@ import { TerminusModule } from '@nestjs/terminus';
|
||||
|
||||
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
|
||||
import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
|
||||
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
|
||||
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
|
||||
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
|
||||
|
||||
import { HealthCacheService } from './health-cache.service';
|
||||
|
||||
@ -12,7 +16,13 @@ import { DatabaseHealthIndicator } from './indicators/database.health';
|
||||
import { RedisHealthIndicator } from './indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from './indicators/worker.health';
|
||||
@Module({
|
||||
imports: [TerminusModule, RedisClientModule],
|
||||
imports: [
|
||||
TerminusModule,
|
||||
RedisClientModule,
|
||||
WorkspaceHealthModule,
|
||||
ObjectMetadataModule,
|
||||
WorkspaceMigrationModule,
|
||||
],
|
||||
controllers: [HealthController, MetricsController],
|
||||
providers: [
|
||||
HealthCacheService,
|
||||
@ -20,6 +30,7 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
|
||||
RedisHealthIndicator,
|
||||
WorkerHealthIndicator,
|
||||
ConnectedAccountHealth,
|
||||
AppHealthIndicator,
|
||||
],
|
||||
exports: [
|
||||
HealthCacheService,
|
||||
@ -27,6 +38,7 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
|
||||
RedisHealthIndicator,
|
||||
WorkerHealthIndicator,
|
||||
ConnectedAccountHealth,
|
||||
AppHealthIndicator,
|
||||
],
|
||||
})
|
||||
export class HealthModule {}
|
||||
|
||||
@ -0,0 +1,167 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
|
||||
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
|
||||
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
|
||||
|
||||
describe('AppHealthIndicator', () => {
|
||||
let service: AppHealthIndicator;
|
||||
let objectMetadataService: jest.Mocked<ObjectMetadataService>;
|
||||
let workspaceHealthService: jest.Mocked<WorkspaceHealthService>;
|
||||
let workspaceMigrationService: jest.Mocked<WorkspaceMigrationService>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
|
||||
beforeEach(async () => {
|
||||
objectMetadataService = {
|
||||
findMany: jest.fn(),
|
||||
} as any;
|
||||
|
||||
workspaceHealthService = {
|
||||
healthCheck: jest.fn(),
|
||||
} as any;
|
||||
|
||||
workspaceMigrationService = {
|
||||
getPendingMigrations: jest.fn(),
|
||||
} as any;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockReturnValue({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
app: { status: 'up', ...data },
|
||||
})),
|
||||
down: jest.fn().mockImplementation((data) => ({
|
||||
app: { status: 'down', ...data },
|
||||
})),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
AppHealthIndicator,
|
||||
{
|
||||
provide: ObjectMetadataService,
|
||||
useValue: objectMetadataService,
|
||||
},
|
||||
{
|
||||
provide: WorkspaceHealthService,
|
||||
useValue: workspaceHealthService,
|
||||
},
|
||||
{
|
||||
provide: WorkspaceMigrationService,
|
||||
useValue: workspaceMigrationService,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<AppHealthIndicator>(AppHealthIndicator);
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return up status when no issues and no pending migrations', async () => {
|
||||
objectMetadataService.findMany.mockResolvedValue([
|
||||
{
|
||||
id: '1',
|
||||
workspaceId: 'workspace1',
|
||||
} as any,
|
||||
{
|
||||
id: '2',
|
||||
workspaceId: 'workspace2',
|
||||
} as any,
|
||||
]);
|
||||
|
||||
workspaceMigrationService.getPendingMigrations.mockResolvedValue([]);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.app.status).toBe('up');
|
||||
expect(result.app.details.overview.totalWorkspacesCount).toBe(2);
|
||||
expect(result.app.details.overview.criticalWorkspacesCount).toBe(0);
|
||||
expect(result.app.details.criticalWorkspaces).toBe(null);
|
||||
expect(result.app.details.system.nodeVersion).toBeDefined();
|
||||
expect(result.app.details.system.timestamp).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return down status when there are pending migrations', async () => {
|
||||
objectMetadataService.findMany.mockResolvedValue([
|
||||
{
|
||||
id: '1',
|
||||
workspaceId: 'workspace1',
|
||||
} as any,
|
||||
]);
|
||||
|
||||
workspaceMigrationService.getPendingMigrations.mockResolvedValue([
|
||||
{
|
||||
id: '1',
|
||||
createdAt: new Date(),
|
||||
migrations: [],
|
||||
name: 'migration1',
|
||||
isCustom: false,
|
||||
workspaceId: 'workspace1',
|
||||
} as any,
|
||||
]);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.app.status).toBe('down');
|
||||
expect(result.app.details.overview.criticalWorkspacesCount).toBe(1);
|
||||
expect(result.app.details.criticalWorkspaces).toEqual([
|
||||
{
|
||||
workspaceId: 'workspace1',
|
||||
pendingMigrations: 1,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should handle errors gracefully and maintain state history', async () => {
|
||||
objectMetadataService.findMany.mockRejectedValue(
|
||||
new Error('Database connection failed'),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.app.status).toBe('down');
|
||||
expect(result.app.message).toBe('Database connection failed');
|
||||
expect(result.app.details.system.nodeVersion).toBeDefined();
|
||||
expect(result.app.details.system.timestamp).toBeDefined();
|
||||
expect(result.app.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should maintain state history across health checks', async () => {
|
||||
// First check - healthy state
|
||||
objectMetadataService.findMany.mockResolvedValue([
|
||||
{
|
||||
id: '1',
|
||||
workspaceId: 'workspace1',
|
||||
} as any,
|
||||
]);
|
||||
workspaceMigrationService.getPendingMigrations.mockResolvedValue([]);
|
||||
|
||||
await service.isHealthy();
|
||||
|
||||
// Second check - error state
|
||||
objectMetadataService.findMany.mockRejectedValue(
|
||||
new Error('Database connection failed'),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.app.details.stateHistory).toBeDefined();
|
||||
expect(result.app.details.stateHistory.age).toBeDefined();
|
||||
expect(result.app.details.stateHistory.timestamp).toBeDefined();
|
||||
expect(result.app.details.stateHistory.details).toBeDefined();
|
||||
});
|
||||
});
|
||||
@ -22,11 +22,8 @@ describe('DatabaseHealthIndicator', () => {
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
database: { status: 'up', ...data },
|
||||
})),
|
||||
down: jest.fn().mockImplementation((error) => ({
|
||||
database: {
|
||||
status: 'down',
|
||||
error,
|
||||
},
|
||||
down: jest.fn().mockImplementation((data) => ({
|
||||
database: { status: 'down', ...data },
|
||||
})),
|
||||
}),
|
||||
} as any;
|
||||
@ -77,10 +74,20 @@ describe('DatabaseHealthIndicator', () => {
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.database.status).toBe('up');
|
||||
expect(result.database.details).toBeDefined();
|
||||
expect(result.database.details.version).toBeDefined();
|
||||
expect(result.database.details.connections).toBeDefined();
|
||||
expect(result.database.details.performance).toBeDefined();
|
||||
expect(result.database.details.system.version).toBe('PostgreSQL 15.6');
|
||||
expect(result.database.details.system.timestamp).toBeDefined();
|
||||
expect(result.database.details.connections).toEqual({
|
||||
active: 5,
|
||||
max: 100,
|
||||
utilizationPercent: 5,
|
||||
});
|
||||
expect(result.database.details.performance).toEqual({
|
||||
cacheHitRatio: '96%',
|
||||
deadlocks: 0,
|
||||
slowQueries: 0,
|
||||
});
|
||||
expect(result.database.details.databaseSize).toBe('1 GB');
|
||||
expect(result.database.details.top10Tables).toEqual([{ table_stats: [] }]);
|
||||
});
|
||||
|
||||
it('should return down status when database fails', async () => {
|
||||
@ -91,9 +98,11 @@ describe('DatabaseHealthIndicator', () => {
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.database.status).toBe('down');
|
||||
expect(result.database.error).toBe(
|
||||
expect(result.database.message).toBe(
|
||||
HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED,
|
||||
);
|
||||
expect(result.database.details.system.timestamp).toBeDefined();
|
||||
expect(result.database.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should timeout after specified duration', async () => {
|
||||
@ -111,6 +120,63 @@ describe('DatabaseHealthIndicator', () => {
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.database.status).toBe('down');
|
||||
expect(result.database.error).toBe(HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT);
|
||||
expect(result.database.message).toBe(
|
||||
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
|
||||
);
|
||||
expect(result.database.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should maintain state history across health checks', async () => {
|
||||
// First check - healthy state
|
||||
const mockResponses = [
|
||||
[{ version: 'PostgreSQL 15.6' }],
|
||||
[{ count: '5' }],
|
||||
[{ max_connections: '100' }],
|
||||
[{ uptime: '3600' }],
|
||||
[{ size: '1 GB' }],
|
||||
[{ table_stats: [] }],
|
||||
[{ ratio: '95.5' }],
|
||||
[{ deadlocks: '0' }],
|
||||
[{ count: '0' }],
|
||||
];
|
||||
|
||||
mockResponses.forEach((response) => {
|
||||
dataSource.query.mockResolvedValueOnce(response);
|
||||
});
|
||||
|
||||
const firstResult = await service.isHealthy();
|
||||
|
||||
expect(firstResult.database.status).toBe('up');
|
||||
|
||||
// Second check - error state
|
||||
dataSource.query.mockRejectedValueOnce(
|
||||
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.database.details.stateHistory).toMatchObject({
|
||||
age: expect.any(Number),
|
||||
timestamp: expect.any(Date),
|
||||
details: {
|
||||
system: {
|
||||
version: 'PostgreSQL 15.6',
|
||||
timestamp: expect.any(String),
|
||||
uptime: expect.any(String),
|
||||
},
|
||||
connections: {
|
||||
active: 5,
|
||||
max: 100,
|
||||
utilizationPercent: 5,
|
||||
},
|
||||
performance: {
|
||||
cacheHitRatio: '96%',
|
||||
deadlocks: 0,
|
||||
slowQueries: 0,
|
||||
},
|
||||
databaseSize: '1 GB',
|
||||
top10Tables: [{ table_stats: [] }],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -35,7 +35,7 @@ describe('RedisHealthIndicator', () => {
|
||||
down: jest.fn().mockImplementation((error) => ({
|
||||
redis: {
|
||||
status: 'down',
|
||||
error,
|
||||
...error,
|
||||
},
|
||||
})),
|
||||
}),
|
||||
@ -72,35 +72,43 @@ describe('RedisHealthIndicator', () => {
|
||||
mockRedis.info
|
||||
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'used_memory_human:1.2G\nused_memory_peak_human:1.5G\nmem_fragmentation_ratio:1.5\n',
|
||||
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
|
||||
)
|
||||
.mockResolvedValueOnce('connected_clients:5\n')
|
||||
.mockResolvedValueOnce('connected_clients:5\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'total_connections_received:100\nkeyspace_hits:90\nkeyspace_misses:10\n',
|
||||
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.redis.status).toBe('up');
|
||||
expect(result.redis.details).toBeDefined();
|
||||
expect(result.redis.details.version).toBe('7.0.0');
|
||||
expect(result.redis.details.system.version).toBe('7.0.0');
|
||||
expect(result.redis.details.system.timestamp).toBeDefined();
|
||||
expect(result.redis.details.memory).toEqual({
|
||||
used: '1.2G',
|
||||
peak: '1.5G',
|
||||
fragmentation: 1.5,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return down status when redis fails', async () => {
|
||||
mockRedis.ping.mockRejectedValueOnce(
|
||||
mockRedis.info.mockRejectedValueOnce(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.redis.status).toBe('down');
|
||||
expect(result.redis.error).toBe(
|
||||
expect(result.redis.message).toBe(
|
||||
HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
|
||||
);
|
||||
expect(result.redis.details.system.timestamp).toBeDefined();
|
||||
expect(result.redis.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should timeout after specified duration', async () => {
|
||||
mockRedis.ping.mockImplementationOnce(
|
||||
mockRedis.info.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
@ -110,24 +118,38 @@ describe('RedisHealthIndicator', () => {
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.redis.status).toBe('down');
|
||||
expect(result.redis.error).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT);
|
||||
expect(result.redis.message).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT);
|
||||
expect(result.redis.details.system.timestamp).toBeDefined();
|
||||
expect(result.redis.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should handle partial failures in health details collection', async () => {
|
||||
it('should maintain state history across health checks', async () => {
|
||||
// First check - healthy state
|
||||
mockRedis.info
|
||||
.mockResolvedValueOnce('redis_version:7.0.0') // info
|
||||
.mockResolvedValueOnce('used_memory_human:1.2G') // memory
|
||||
.mockResolvedValueOnce('connected_clients:5') // clients
|
||||
.mockResolvedValueOnce('total_connections_received:100'); // stats
|
||||
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
|
||||
)
|
||||
.mockResolvedValueOnce('connected_clients:5\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
|
||||
);
|
||||
|
||||
await service.isHealthy();
|
||||
|
||||
// Second check - error state
|
||||
mockRedis.info.mockRejectedValueOnce(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.redis.status).toBe('up');
|
||||
expect(result.redis.details).toBeDefined();
|
||||
expect(result.redis.details.version).toBe('7.0.0');
|
||||
expect(result.redis.details.stateHistory).toBeDefined();
|
||||
expect(result.redis.details.stateHistory.age).toBeDefined();
|
||||
expect(result.redis.details.stateHistory.timestamp).toBeDefined();
|
||||
expect(result.redis.details.stateHistory.details).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
@ -0,0 +1,96 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
|
||||
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
|
||||
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
|
||||
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
|
||||
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
|
||||
|
||||
@Injectable()
|
||||
export class AppHealthIndicator {
|
||||
private stateManager = new HealthStateManager();
|
||||
|
||||
constructor(
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
private readonly workspaceHealthService: WorkspaceHealthService,
|
||||
private readonly objectMetadataService: ObjectMetadataService,
|
||||
private readonly workspaceMigrationService: WorkspaceMigrationService,
|
||||
) {}
|
||||
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('app');
|
||||
|
||||
try {
|
||||
const workspaces = await this.objectMetadataService.findMany();
|
||||
const workspaceIds = [...new Set(workspaces.map((w) => w.workspaceId))];
|
||||
|
||||
const workspaceStats = await Promise.all(
|
||||
workspaceIds.map(async (workspaceId) => {
|
||||
const pendingMigrations =
|
||||
await this.workspaceMigrationService.getPendingMigrations(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return {
|
||||
workspaceId,
|
||||
pendingMigrations: pendingMigrations.length,
|
||||
isCritical: pendingMigrations.length > 0,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
const details = {
|
||||
system: {
|
||||
nodeVersion: process.version,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
overview: {
|
||||
totalWorkspacesCount: workspaceIds.length,
|
||||
criticalWorkspacesCount: workspaceStats.filter(
|
||||
(stat) => stat.isCritical,
|
||||
).length,
|
||||
},
|
||||
criticalWorkspaces:
|
||||
workspaceStats.filter((stat) => stat.isCritical).length > 0
|
||||
? workspaceStats
|
||||
.filter((stat) => stat.isCritical)
|
||||
.map((stat) => ({
|
||||
workspaceId: stat.workspaceId,
|
||||
pendingMigrations: stat.pendingMigrations,
|
||||
}))
|
||||
: null,
|
||||
};
|
||||
|
||||
const isHealthy = workspaceStats.every((stat) => !stat.isCritical);
|
||||
|
||||
if (isHealthy) {
|
||||
this.stateManager.updateState(details);
|
||||
|
||||
return indicator.up({ details });
|
||||
}
|
||||
|
||||
this.stateManager.updateState(details);
|
||||
|
||||
return indicator.down({
|
||||
message: `Found ${details.criticalWorkspaces?.length} workspaces with pending migrations`,
|
||||
details,
|
||||
});
|
||||
} catch (error) {
|
||||
const stateWithAge = this.stateManager.getStateWithAge();
|
||||
|
||||
return indicator.down({
|
||||
message: error.message,
|
||||
details: {
|
||||
system: {
|
||||
nodeVersion: process.version,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
stateHistory: stateWithAge,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -9,9 +9,12 @@ import { DataSource } from 'typeorm';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
|
||||
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
|
||||
|
||||
@Injectable()
|
||||
export class DatabaseHealthIndicator {
|
||||
private stateManager = new HealthStateManager();
|
||||
|
||||
constructor(
|
||||
@InjectDataSource('core')
|
||||
private readonly dataSource: DataSource,
|
||||
@ -69,35 +72,50 @@ export class DatabaseHealthIndicator {
|
||||
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
|
||||
);
|
||||
|
||||
return indicator.up({
|
||||
details: {
|
||||
const details = {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
version: versionResult.version,
|
||||
connections: {
|
||||
active: parseInt(activeConnections.count),
|
||||
max: parseInt(maxConnections.max_connections),
|
||||
utilizationPercent: Math.round(
|
||||
(parseInt(activeConnections.count) /
|
||||
parseInt(maxConnections.max_connections)) *
|
||||
100,
|
||||
),
|
||||
},
|
||||
uptime: Math.round(uptime.uptime / 3600) + ' hours',
|
||||
databaseSize: databaseSize.size,
|
||||
performance: {
|
||||
cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%',
|
||||
deadlocks: parseInt(deadlocks.deadlocks),
|
||||
slowQueries: parseInt(slowQueries.count),
|
||||
},
|
||||
top10Tables: tableStats,
|
||||
},
|
||||
});
|
||||
connections: {
|
||||
active: parseInt(activeConnections.count),
|
||||
max: parseInt(maxConnections.max_connections),
|
||||
utilizationPercent: Math.round(
|
||||
(parseInt(activeConnections.count) /
|
||||
parseInt(maxConnections.max_connections)) *
|
||||
100,
|
||||
),
|
||||
},
|
||||
databaseSize: databaseSize.size,
|
||||
performance: {
|
||||
cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%',
|
||||
deadlocks: parseInt(deadlocks.deadlocks),
|
||||
slowQueries: parseInt(slowQueries.count),
|
||||
},
|
||||
top10Tables: tableStats,
|
||||
};
|
||||
|
||||
this.stateManager.updateState(details);
|
||||
|
||||
return indicator.up({ details });
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
const message =
|
||||
error.message === HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED;
|
||||
|
||||
return indicator.down(errorMessage);
|
||||
const stateWithAge = this.stateManager.getStateWithAge();
|
||||
|
||||
return indicator.down({
|
||||
message,
|
||||
details: {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
stateHistory: stateWithAge,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,10 +6,13 @@ import {
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
|
||||
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
@Injectable()
|
||||
export class RedisHealthIndicator {
|
||||
private stateManager = new HealthStateManager();
|
||||
|
||||
constructor(
|
||||
private readonly redisClient: RedisClientService,
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
@ -48,47 +51,62 @@ export class RedisHealthIndicator {
|
||||
const clientsData = parseInfo(clients);
|
||||
const statsData = parseInfo(stats);
|
||||
|
||||
return indicator.up({
|
||||
details: {
|
||||
const details = {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
version: infoData.redis_version,
|
||||
uptime:
|
||||
Math.round(parseInt(infoData.uptime_in_seconds) / 3600) + ' hours',
|
||||
memory: {
|
||||
used: memoryData.used_memory_human,
|
||||
peak: memoryData.used_memory_peak_human,
|
||||
fragmentation: parseFloat(memoryData.mem_fragmentation_ratio),
|
||||
},
|
||||
connections: {
|
||||
current: parseInt(clientsData.connected_clients),
|
||||
total: parseInt(statsData.total_connections_received),
|
||||
rejected: parseInt(statsData.rejected_connections),
|
||||
},
|
||||
performance: {
|
||||
opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec),
|
||||
hitRate: statsData.keyspace_hits
|
||||
? Math.round(
|
||||
(parseInt(statsData.keyspace_hits) /
|
||||
(parseInt(statsData.keyspace_hits) +
|
||||
parseInt(statsData.keyspace_misses))) *
|
||||
100,
|
||||
) + '%'
|
||||
: '0%',
|
||||
evictedKeys: parseInt(statsData.evicted_keys),
|
||||
expiredKeys: parseInt(statsData.expired_keys),
|
||||
},
|
||||
replication: {
|
||||
role: infoData.role,
|
||||
connectedSlaves: parseInt(infoData.connected_slaves || '0'),
|
||||
},
|
||||
},
|
||||
});
|
||||
memory: {
|
||||
used: memoryData.used_memory_human,
|
||||
peak: memoryData.used_memory_peak_human,
|
||||
fragmentation: parseFloat(memoryData.mem_fragmentation_ratio),
|
||||
},
|
||||
connections: {
|
||||
current: parseInt(clientsData.connected_clients),
|
||||
total: parseInt(statsData.total_connections_received),
|
||||
rejected: parseInt(statsData.rejected_connections),
|
||||
},
|
||||
performance: {
|
||||
opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec),
|
||||
hitRate: statsData.keyspace_hits
|
||||
? Math.round(
|
||||
(parseInt(statsData.keyspace_hits) /
|
||||
(parseInt(statsData.keyspace_hits) +
|
||||
parseInt(statsData.keyspace_misses))) *
|
||||
100,
|
||||
) + '%'
|
||||
: '0%',
|
||||
evictedKeys: parseInt(statsData.evicted_keys),
|
||||
expiredKeys: parseInt(statsData.expired_keys),
|
||||
},
|
||||
replication: {
|
||||
role: infoData.role,
|
||||
connectedSlaves: parseInt(infoData.connected_slaves || '0'),
|
||||
},
|
||||
};
|
||||
|
||||
this.stateManager.updateState(details);
|
||||
|
||||
return indicator.up({ details });
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
const message =
|
||||
error.message === HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED;
|
||||
|
||||
return indicator.down(errorMessage);
|
||||
const stateWithAge = this.stateManager.getStateWithAge();
|
||||
|
||||
return indicator.down({
|
||||
message,
|
||||
details: {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
stateHistory: stateWithAge,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
export class HealthStateManager {
|
||||
private lastKnownState: {
|
||||
timestamp: Date;
|
||||
details: Record<string, any>;
|
||||
} | null = null;
|
||||
|
||||
updateState(details: Record<string, any>) {
|
||||
this.lastKnownState = {
|
||||
timestamp: new Date(),
|
||||
details,
|
||||
};
|
||||
}
|
||||
|
||||
getStateWithAge() {
|
||||
return this.lastKnownState
|
||||
? {
|
||||
...this.lastKnownState,
|
||||
age: Date.now() - this.lastKnownState.timestamp.getTime(),
|
||||
}
|
||||
: 'No previous state available';
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user