Admin panel: App health check (#10546)

closes https://github.com/twentyhq/core-team-issues/issues/441
whats new - 
- app health with the proposed format -- update, scratched this format
in favor of a more concise and more intentional health check. Now we
will check app health only based on pendingMigrations
<del>
```
status: 
  system: {
    nodeVersion
  },
  overview: {
    totalWorkspaces,
    criticalWorkspaces,
    workspacesWithPendingMigrations,
    healthDistribution: {
      healthy,
      warning,
      critical,
    }
  },
 problematicWorkspaces: [
    {
      workspaceId,
      severity,
      pendingMigrations,
      issuesSummary: {
        structural,
        data,
        relationship
      }
    }
  ]
  ```
</del>

- errorMessage and details seperation -- before we used to send error in details which made it difficult if we want both error and details on the front -- usecase >>
suppose app health indicator is not healthy but still want to send details
- stateHistory with timestamp -- this is something I introduced, not sure about this. Basically the thought process was to store the LastState of the details, incase of no connection or timeout errors. This is not yet used on the front, just the endpoint. But it could be used on the front too
- name unifying ⁠https://discord.com/channels/1130383047699738754/1346454192776155156
- json tree https://discord.com/channels/1130383047699738754/1346458558048501760
- match figma design https://discord.com/channels/1130383047699738754/1346451659647094815
- fix the collapse/open styles in tables https://discord.com/channels/1130383047699738754/1346452051974160406
- shift eye icon to expanded container https://discord.com/channels/1130383047699738754/1346452282669010987
- use H2Title for title and description of env variables groups https://discord.com/channels/1130383047699738754/1346434955936530452
This commit is contained in:
nitin
2025-03-07 01:49:41 +05:30
committed by GitHub
parent f6314e52fe
commit 3061576302
34 changed files with 808 additions and 244 deletions

View File

@ -30,6 +30,7 @@ export type AdminPanelHealthServiceData = {
__typename?: 'AdminPanelHealthServiceData';
description: Scalars['String'];
details?: Maybe<Scalars['String']>;
errorMessage?: Maybe<Scalars['String']>;
id: Scalars['String'];
label: Scalars['String'];
queues?: Maybe<Array<AdminPanelWorkerQueueHealth>>;
@ -642,6 +643,7 @@ export type GlobalSearchRecord = {
};
export enum HealthIndicatorId {
app = 'app',
connectedAccount = 'connectedAccount',
database = 'database',
redis = 'redis',
@ -2455,7 +2457,7 @@ export type GetIndicatorHealthStatusQueryVariables = Exact<{
}>;
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus }> | null } };
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, errorMessage?: string | null, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus }> | null } };
export type GetQueueMetricsQueryVariables = Exact<{
queueName: Scalars['String'];
@ -4305,6 +4307,7 @@ export const GetIndicatorHealthStatusDocument = gql`
label
description
status
errorMessage
details
queues {
id

View File

@ -450,13 +450,13 @@ export const SettingsRoutes = ({
{isAdminPageEnabled && (
<>
<Route path={SettingsPath.AdminPanel} element={<SettingsAdmin />} />
<Route path={SettingsPath.ServerAdmin} element={<SettingsAdmin />} />
<Route
path={SettingsPath.FeatureFlags}
element={<SettingsAdminContent />}
/>
<Route
path={SettingsPath.AdminPanelIndicatorHealthStatus}
path={SettingsPath.ServerAdminIndicatorHealthStatus}
element={<SettingsAdminIndicatorHealthStatus />}
/>
</>

View File

@ -3,8 +3,8 @@ import { SETTINGS_ADMIN_TABS } from '@/settings/admin-panel/constants/SettingsAd
import { SETTINGS_ADMIN_TABS_ID } from '@/settings/admin-panel/constants/SettingsAdminTabsId';
import { TabList } from '@/ui/layout/tab/components/TabList';
import styled from '@emotion/styled';
import { IconHeart, IconSettings2, IconVariable } from 'twenty-ui';
import { t } from '@lingui/core/macro';
import { IconHeart, IconSettings2, IconVariable } from 'twenty-ui';
const StyledTabListContainer = styled.div`
align-items: center;

View File

@ -2,22 +2,13 @@ import { SettingsAdminEnvVariablesTable } from '@/settings/admin-panel/component
import { SettingsAdminTabSkeletonLoader } from '@/settings/admin-panel/components/SettingsAdminTabSkeletonLoader';
import styled from '@emotion/styled';
import { useState } from 'react';
import { Button, H1Title, H1TitleFontColor, Section } from 'twenty-ui';
import { Button, H1Title, H1TitleFontColor, H2Title, Section } from 'twenty-ui';
import { useGetEnvironmentVariablesGroupedQuery } from '~/generated/graphql';
const StyledGroupContainer = styled.div`
margin-bottom: ${({ theme }) => theme.spacing(6)};
`;
const StyledGroupVariablesContainer = styled.div`
background-color: ${({ theme }) => theme.background.secondary};
border-radius: ${({ theme }) => theme.border.radius.sm};
border: 1px solid ${({ theme }) => theme.border.color.medium};
padding-bottom: ${({ theme }) => theme.spacing(2)};
padding-left: ${({ theme }) => theme.spacing(4)};
padding-right: ${({ theme }) => theme.spacing(4)};
`;
const StyledGroupDescription = styled.div`
margin-bottom: ${({ theme }) => theme.spacing(4)};
`;
@ -78,16 +69,9 @@ export const SettingsAdminEnvVariables = () => {
<Section>
{visibleGroups.map((group) => (
<StyledGroupContainer key={group.name}>
<H1Title title={group.name} fontColor={H1TitleFontColor.Primary} />
{group.description !== '' && (
<StyledGroupDescription>
{group.description}
</StyledGroupDescription>
)}
<H2Title title={group.name} description={group.description} />
{group.variables.length > 0 && (
<StyledGroupVariablesContainer>
<SettingsAdminEnvVariablesTable variables={group.variables} />
</StyledGroupVariablesContainer>
<SettingsAdminEnvVariablesTable variables={group.variables} />
)}
</StyledGroupContainer>
))}
@ -120,11 +104,9 @@ export const SettingsAdminEnvVariables = () => {
</StyledGroupDescription>
)}
{selectedGroupData.variables.length > 0 && (
<StyledGroupVariablesContainer>
<SettingsAdminEnvVariablesTable
variables={selectedGroupData.variables}
/>
</StyledGroupVariablesContainer>
<SettingsAdminEnvVariablesTable
variables={selectedGroupData.variables}
/>
)}
</StyledGroupContainer>
)}

View File

@ -2,6 +2,7 @@ import { TableCell } from '@/ui/layout/table/components/TableCell';
import { TableRow } from '@/ui/layout/table/components/TableRow';
import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { motion } from 'framer-motion';
import { useState } from 'react';
import {
AnimatedExpandableContainer,
@ -10,6 +11,7 @@ import {
IconEyeOff,
LightIconButton,
} from 'twenty-ui';
type SettingsAdminEnvVariablesRowProps = {
variable: {
name: string;
@ -26,8 +28,23 @@ const StyledTruncatedCell = styled(TableCell)`
cursor: pointer;
`;
const StyledButton = styled(motion.button)`
align-items: center;
border: none;
display: flex;
justify-content: center;
padding-inline: ${({ theme }) => theme.spacing(1)};
background-color: transparent;
height: 24px;
width: 24px;
box-sizing: border-box;
cursor: pointer;
`;
const MotionIconChevronDown = motion(IconChevronRight);
const StyledExpandedDetails = styled.div`
background-color: ${({ theme }) => theme.background.tertiary};
background-color: ${({ theme }) => theme.background.secondary};
border-radius: ${({ theme }) => theme.border.radius.sm};
margin: ${({ theme }) => theme.spacing(2)} 0;
padding: ${({ theme }) => theme.spacing(2)};
@ -40,7 +57,8 @@ const StyledExpandedDetails = styled.div`
`;
const StyledDetailLabel = styled.div`
font-weight: ${({ theme }) => theme.font.weight.medium};
color: ${({ theme }) => theme.font.color.tertiary};
font-weight: ${({ theme }) => theme.font.weight.regular};
padding-right: ${({ theme }) => theme.spacing(4)};
`;
@ -56,12 +74,16 @@ const StyledExpandedLabel = styled.div`
overflow: visible;
`;
const StyledTransitionedIconChevronRight = styled(IconChevronRight)`
cursor: pointer;
transform: ${({ $isExpanded }: { $isExpanded: boolean }) =>
$isExpanded ? 'rotate(90deg)' : 'rotate(0deg)'};
transition: ${({ theme }) =>
`transform ${theme.animation.duration.normal}s ease`};
const StyledValueContainer = styled.div`
display: flex;
align-items: center;
justify-content: space-between;
`;
const StyledTableRow = styled(TableRow)<{ isExpanded: boolean }>`
background-color: ${({ isExpanded, theme }) =>
isExpanded ? theme.background.transparent.light : 'transparent'};
margin-bottom: ${({ theme }) => theme.spacing(0.5)};
`;
export const SettingsAdminEnvVariablesRow = ({
@ -85,44 +107,51 @@ export const SettingsAdminEnvVariablesRow = ({
return (
<>
<TableRow
<StyledTableRow
onClick={() => setIsExpanded(!isExpanded)}
gridAutoColumns="4fr 3fr 2fr 1fr 1fr"
gridAutoColumns="5fr 4fr 3fr 1fr"
isExpanded={isExpanded}
>
<StyledTruncatedCell color="primary">
<StyledTruncatedCell color={theme.font.color.primary}>
<StyledEllipsisLabel>{variable.name}</StyledEllipsisLabel>
</StyledTruncatedCell>
<StyledTruncatedCell>
<StyledEllipsisLabel>{variable.description}</StyledEllipsisLabel>
</StyledTruncatedCell>
<StyledTruncatedCell>
<StyledTruncatedCell align="right">
<StyledEllipsisLabel>{displayValue}</StyledEllipsisLabel>
</StyledTruncatedCell>
<TableCell align="right">
{variable.sensitive && variable.value !== '' && (
<LightIconButton
Icon={showSensitiveValue ? IconEyeOff : IconEye}
size="small"
accent="secondary"
onClick={handleToggleVisibility}
<StyledButton onClick={() => setIsExpanded(!isExpanded)}>
<MotionIconChevronDown
size={theme.icon.size.md}
color={theme.font.color.tertiary}
initial={false}
animate={{ rotate: isExpanded ? 90 : 0 }}
/>
)}
</StyledButton>
</TableCell>
<TableCell align="right">
<StyledTransitionedIconChevronRight
$isExpanded={isExpanded}
size={theme.icon.size.sm}
/>
</TableCell>
</TableRow>
</StyledTableRow>
<AnimatedExpandableContainer isExpanded={isExpanded} mode="fit-content">
<StyledExpandedDetails>
<StyledDetailLabel>Name:</StyledDetailLabel>
<StyledDetailLabel>Name</StyledDetailLabel>
<StyledEllipsisLabel>{variable.name}</StyledEllipsisLabel>
<StyledDetailLabel>Description:</StyledDetailLabel>
<StyledDetailLabel>Description</StyledDetailLabel>
<StyledExpandedLabel>{variable.description}</StyledExpandedLabel>
<StyledDetailLabel>Value:</StyledDetailLabel>
<StyledExpandedLabel>{displayValue}</StyledExpandedLabel>
<StyledDetailLabel>Value</StyledDetailLabel>
<StyledExpandedLabel>
<StyledValueContainer>
{displayValue}
{variable.sensitive && variable.value !== '' && (
<LightIconButton
Icon={showSensitiveValue ? IconEyeOff : IconEye}
size="small"
accent="secondary"
onClick={handleToggleVisibility}
/>
)}
</StyledValueContainer>
</StyledExpandedLabel>
</StyledExpandedDetails>
</AnimatedExpandableContainer>
</>

View File

@ -21,11 +21,10 @@ export const SettingsAdminEnvVariablesTable = ({
variables,
}: SettingsAdminEnvVariablesTableProps) => (
<StyledTable>
<TableRow gridAutoColumns="4fr 3fr 2fr 1fr 1fr">
<TableRow gridAutoColumns="5fr 4fr 3fr 1fr">
<TableHeader>Name</TableHeader>
<TableHeader>Description</TableHeader>
<TableHeader>Value</TableHeader>
<TableHeader align="right"></TableHeader>
<TableHeader align="right">Value</TableHeader>
<TableHeader align="right"></TableHeader>
</TableRow>
{variables.map((variable) => (

View File

@ -17,11 +17,13 @@ export const ConnectedAccountHealthStatus = () => {
}
const parsedDetails = JSON.parse(details);
const serviceDetails = parsedDetails.details;
const isMessageSyncDown =
parsedDetails.messageSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
serviceDetails.messageSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
const isCalendarSyncDown =
parsedDetails.calendarSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
serviceDetails.calendarSync?.status ===
AdminPanelHealthServiceStatus.OUTAGE;
const errorMessages = [];
if (isMessageSyncDown) {
@ -39,16 +41,16 @@ export const ConnectedAccountHealthStatus = () => {
</StyledErrorMessage>
)}
{!isMessageSyncDown && parsedDetails.messageSync?.details && (
{!isMessageSyncDown && serviceDetails.messageSync?.details && (
<SettingsAdminHealthAccountSyncCountersTable
details={parsedDetails.messageSync.details}
details={serviceDetails.messageSync.details}
title="Message Sync Status"
/>
)}
{!isCalendarSyncDown && parsedDetails.calendarSync?.details && (
{!isCalendarSyncDown && serviceDetails.calendarSync?.details && (
<SettingsAdminHealthAccountSyncCountersTable
details={parsedDetails.calendarSync.details}
details={serviceDetails.calendarSync.details}
title="Calendar Sync Status"
/>
)}

View File

@ -1,43 +0,0 @@
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
import styled from '@emotion/styled';
import { useContext } from 'react';
import { Section } from 'twenty-ui';
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
const StyledDetailsContainer = styled.pre`
background-color: ${({ theme }) => theme.background.quaternary};
padding: ${({ theme }) => theme.spacing(6)};
border-radius: ${({ theme }) => theme.border.radius.sm};
white-space: pre-wrap;
font-size: ${({ theme }) => theme.font.size.sm};
margin: 0;
`;
const StyledErrorMessage = styled.div`
color: ${({ theme }) => theme.color.red};
margin-top: ${({ theme }) => theme.spacing(2)};
`;
export const DatabaseAndRedisHealthStatus = () => {
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
const formattedDetails = indicatorHealth.details
? JSON.stringify(JSON.parse(indicatorHealth.details), null, 2)
: null;
const isDatabaseOrRedisDown =
!indicatorHealth.status ||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
return (
<Section>
{isDatabaseOrRedisDown ? (
<StyledErrorMessage>
{`${indicatorHealth.label} information is not available because the service is down`}
</StyledErrorMessage>
) : (
<StyledDetailsContainer>{formattedDetails}</StyledDetailsContainer>
)}
</Section>
);
};

View File

@ -0,0 +1,49 @@
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
import { JsonTree } from '@/workflow/components/json-visualizer/components/JsonTree';
import styled from '@emotion/styled';
import { useContext } from 'react';
import { Section } from 'twenty-ui';
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
const StyledDetailsContainer = styled.div`
background-color: ${({ theme }) => theme.background.secondary};
padding: ${({ theme }) => theme.spacing(4)};
border-radius: ${({ theme }) => theme.border.radius.md};
border: 1px solid ${({ theme }) => theme.border.color.medium};
white-space: pre-wrap;
font-size: ${({ theme }) => theme.font.size.sm};
`;
const StyledErrorMessage = styled.div`
color: ${({ theme }) => theme.color.red};
margin-top: ${({ theme }) => theme.spacing(2)};
margin-bottom: ${({ theme }) => theme.spacing(4)};
`;
export const JsonDataIndicatorHealthStatus = () => {
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
const parsedDetails = indicatorHealth.details
? JSON.parse(indicatorHealth.details)
: null;
const isDown =
!indicatorHealth.status ||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
return (
<Section>
{isDown && (
<StyledErrorMessage>
{indicatorHealth.errorMessage ||
`${indicatorHealth.label} service is unreachable`}
</StyledErrorMessage>
)}
{parsedDetails && (
<StyledDetailsContainer>
<JsonTree value={parsedDetails} />
</StyledDetailsContainer>
)}
</Section>
);
};

View File

@ -1,5 +1,5 @@
import { ConnectedAccountHealthStatus } from '@/settings/admin-panel/health-status/components/ConnectedAccountHealthStatus';
import { DatabaseAndRedisHealthStatus } from '@/settings/admin-panel/health-status/components/DatabaseAndRedisHealthStatus';
import { JsonDataIndicatorHealthStatus } from '@/settings/admin-panel/health-status/components/JsonDataIndicatorHealthStatus';
import { WorkerHealthStatus } from '@/settings/admin-panel/health-status/components/WorkerHealthStatus';
import { useParams } from 'react-router-dom';
import { HealthIndicatorId } from '~/generated/graphql';
@ -10,11 +10,13 @@ export const SettingsAdminIndicatorHealthStatusContent = () => {
switch (indicatorId) {
case HealthIndicatorId.database:
case HealthIndicatorId.redis:
return <DatabaseAndRedisHealthStatus />;
case HealthIndicatorId.app:
return <JsonDataIndicatorHealthStatus />;
case HealthIndicatorId.worker:
return <WorkerHealthStatus />;
case HealthIndicatorId.connectedAccount:
return <ConnectedAccountHealthStatus />;
default:
return null;
}

View File

@ -20,7 +20,7 @@ export const SettingsHealthStatusListCard = ({
<SettingsAdminHealthStatusRightContainer status={service.status} />
)}
to={(service) =>
getSettingsPath(SettingsPath.AdminPanelIndicatorHealthStatus, {
getSettingsPath(SettingsPath.ServerAdminIndicatorHealthStatus, {
indicatorId: service.id,
})
}

View File

@ -14,6 +14,7 @@ export const SettingsAdminIndicatorHealthContext =
id: '',
label: '',
description: '',
errorMessage: '',
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: '',
queues: [],

View File

@ -7,6 +7,7 @@ export const GET_INDICATOR_HEALTH_STATUS = gql`
label
description
status
errorMessage
details
queues {
id

View File

@ -179,7 +179,7 @@ const useSettingsNavigationItems = (): SettingsNavigationSection[] => {
items: [
{
label: t`Server Admin`,
path: SettingsPath.AdminPanel,
path: SettingsPath.ServerAdmin,
Icon: IconServer,
isHidden: !isAdminEnabled,
},

View File

@ -34,10 +34,10 @@ export enum SettingsPath {
DevelopersNewWebhook = 'developers/webhooks/new',
DevelopersNewWebhookDetail = 'developers/webhooks/:webhookId',
Releases = 'releases',
AdminPanel = 'admin-panel',
FeatureFlags = 'admin-panel/feature-flags',
AdminPanelHealthStatus = 'admin-panel#health-status',
AdminPanelIndicatorHealthStatus = 'admin-panel/health-status/:indicatorId',
ServerAdmin = 'server-admin',
FeatureFlags = 'server-admin/feature-flags',
ServerAdminHealthStatus = 'server-admin#health-status',
ServerAdminIndicatorHealthStatus = 'server-admin/health-status/:indicatorId',
Lab = 'lab',
Roles = 'roles',
RoleDetail = 'roles/:roleId',

View File

@ -14,9 +14,9 @@ export const SettingsAdmin = () => {
links={[
{
children: t`Other`,
href: getSettingsPath(SettingsPath.AdminPanel),
href: getSettingsPath(SettingsPath.ServerAdmin),
},
{ children: t`Server Admin Panel` },
{ children: t`Server Admin` },
]}
>
<SettingsPageContainer>

View File

@ -20,6 +20,17 @@ const StyledH2Title = styled(H2Title)`
margin-top: ${({ theme }) => theme.spacing(2)};
`;
const StyledTitleContainer = styled.div`
display: flex;
align-items: center;
gap: ${({ theme }) => theme.spacing(2)};
`;
const StyledHealthStatusContainer = styled.div`
margin-bottom: ${({ theme }) => theme.spacing(4)};
margin-top: ${({ theme }) => theme.spacing(1)};
`;
export const SettingsAdminIndicatorHealthStatus = () => {
const { t } = useLingui();
const { indicatorId } = useParams();
@ -40,15 +51,15 @@ export const SettingsAdminIndicatorHealthStatus = () => {
links={[
{
children: t`Other`,
href: getSettingsPath(SettingsPath.AdminPanel),
href: getSettingsPath(SettingsPath.ServerAdmin),
},
{
children: t`Server Admin Panel`,
href: getSettingsPath(SettingsPath.AdminPanel),
children: t`Server Admin`,
href: getSettingsPath(SettingsPath.ServerAdmin),
},
{
children: t`Health Status`,
href: getSettingsPath(SettingsPath.AdminPanelHealthStatus),
href: getSettingsPath(SettingsPath.ServerAdminHealthStatus),
},
{ children: `${data?.getIndicatorHealthStatus?.label}` },
]}
@ -60,6 +71,7 @@ export const SettingsAdminIndicatorHealthStatus = () => {
id: data?.getIndicatorHealthStatus?.id ?? '',
label: data?.getIndicatorHealthStatus?.label ?? '',
description: data?.getIndicatorHealthStatus?.description ?? '',
errorMessage: data?.getIndicatorHealthStatus?.errorMessage,
status:
data?.getIndicatorHealthStatus?.status ??
AdminPanelHealthServiceStatus.OUTAGE,
@ -69,16 +81,20 @@ export const SettingsAdminIndicatorHealthStatus = () => {
}}
>
<Section>
<StyledH2Title
title={`${data?.getIndicatorHealthStatus?.label}`}
description={data?.getIndicatorHealthStatus?.description}
/>
{indicatorId !== HealthIndicatorId.connectedAccount &&
data?.getIndicatorHealthStatus?.status && (
<SettingsAdminHealthStatusRightContainer
status={data?.getIndicatorHealthStatus.status}
/>
)}
<StyledTitleContainer>
<StyledH2Title
title={`${data?.getIndicatorHealthStatus?.label}`}
description={data?.getIndicatorHealthStatus?.description}
/>
{indicatorId !== HealthIndicatorId.connectedAccount &&
data?.getIndicatorHealthStatus?.status && (
<StyledHealthStatusContainer>
<SettingsAdminHealthStatusRightContainer
status={data?.getIndicatorHealthStatus.status}
/>
</StyledHealthStatusContainer>
)}
</StyledTitleContainer>
</Section>
<SettingsAdminIndicatorHealthStatusContent />

View File

@ -11,6 +11,7 @@ import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
@ -26,6 +27,7 @@ describe('AdminPanelHealthService', () => {
let redisHealth: jest.Mocked<RedisHealthIndicator>;
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
let appHealth: jest.Mocked<AppHealthIndicator>;
let redisClient: jest.Mocked<RedisClientService>;
let environmentService: jest.Mocked<EnvironmentService>;
let loggerSpy: jest.SpyInstance;
@ -35,6 +37,7 @@ describe('AdminPanelHealthService', () => {
redisHealth = { isHealthy: jest.fn() } as any;
workerHealth = { isHealthy: jest.fn(), getQueueDetails: jest.fn() } as any;
connectedAccountHealth = { isHealthy: jest.fn() } as any;
appHealth = { isHealthy: jest.fn() } as any;
redisClient = {
getClient: jest.fn().mockReturnValue({} as Redis),
} as any;
@ -53,6 +56,7 @@ describe('AdminPanelHealthService', () => {
{ provide: RedisHealthIndicator, useValue: redisHealth },
{ provide: WorkerHealthIndicator, useValue: workerHealth },
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
{ provide: AppHealthIndicator, useValue: appHealth },
{ provide: RedisClientService, useValue: redisClient },
{ provide: EnvironmentService, useValue: environmentService },
],
@ -108,6 +112,32 @@ describe('AdminPanelHealthService', () => {
details: 'Account sync is operational',
},
});
appHealth.isHealthy.mockResolvedValue({
app: {
status: 'up',
details: {
system: {
nodeVersion: '16.0',
},
workspaces: {
totalWorkspaces: 1,
healthStatus: [
{
workspaceId: '1',
summary: {
structuralIssues: 0,
dataIssues: 0,
relationshipIssues: 0,
pendingMigrations: 0,
},
severity: 'healthy',
details: {},
},
],
},
},
},
});
const result = await service.getSystemHealthStatus();
@ -129,6 +159,10 @@ describe('AdminPanelHealthService', () => {
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
],
};
@ -148,6 +182,9 @@ describe('AdminPanelHealthService', () => {
connectedAccountHealth.isHealthy.mockResolvedValue({
connectedAccount: { status: 'up' },
});
appHealth.isHealthy.mockResolvedValue({
app: { status: 'down', details: {} },
});
const result = await service.getSystemHealthStatus();
@ -169,6 +206,10 @@ describe('AdminPanelHealthService', () => {
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
],
});
});
@ -186,6 +227,9 @@ describe('AdminPanelHealthService', () => {
connectedAccountHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED),
);
appHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.APP_HEALTH_CHECK_FAILED),
);
const result = await service.getSystemHealthStatus();
@ -207,6 +251,10 @@ describe('AdminPanelHealthService', () => {
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
],
});
});
@ -233,7 +281,8 @@ describe('AdminPanelHealthService', () => {
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.database],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: JSON.stringify(details),
details: JSON.stringify({ details }),
errorMessage: undefined,
queues: undefined,
});
});
@ -266,7 +315,8 @@ describe('AdminPanelHealthService', () => {
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.worker],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: undefined,
details: JSON.stringify({ queues: mockQueues }),
errorMessage: undefined,
queues: mockQueues.map((queue) => ({
id: `worker-${queue.queueName}`,
queueName: queue.queueName,
@ -290,7 +340,8 @@ describe('AdminPanelHealthService', () => {
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.redis],
status: AdminPanelHealthServiceStatus.OUTAGE,
details: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
details: undefined,
errorMessage: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
});
});

View File

@ -10,6 +10,7 @@ import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-he
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
@ -27,6 +28,7 @@ export class AdminPanelHealthService {
private readonly redisHealth: RedisHealthIndicator,
private readonly workerHealth: WorkerHealthIndicator,
private readonly connectedAccountHealth: ConnectedAccountHealth,
private readonly appHealth: AppHealthIndicator,
private readonly redisClient: RedisClientService,
) {}
@ -35,6 +37,7 @@ export class AdminPanelHealthService {
[HealthIndicatorId.redis]: this.redisHealth,
[HealthIndicatorId.worker]: this.workerHealth,
[HealthIndicatorId.connectedAccount]: this.connectedAccountHealth,
[HealthIndicatorId.app]: this.appHealth,
};
private transformStatus(status: HealthIndicatorStatus) {
@ -46,15 +49,17 @@ export class AdminPanelHealthService {
private transformServiceDetails(details: any) {
if (!details) return details;
if (details.messageSync) {
details.messageSync.status = this.transformStatus(
details.messageSync.status,
);
}
if (details.calendarSync) {
details.calendarSync.status = this.transformStatus(
details.calendarSync.status,
);
if (details.details) {
if (details.details.messageSync) {
details.details.messageSync.status = this.transformStatus(
details.details.messageSync.status,
);
}
if (details.details.calendarSync) {
details.details.calendarSync.status = this.transformStatus(
details.details.calendarSync.status,
);
}
}
return details;
@ -65,17 +70,33 @@ export class AdminPanelHealthService {
indicatorId: HealthIndicatorId,
) {
if (result.status === 'fulfilled') {
const key = Object.keys(result.value)[0];
const keys = Object.keys(result.value);
if (keys.length === 0) {
return {
...HEALTH_INDICATORS[indicatorId],
status: AdminPanelHealthServiceStatus.OUTAGE,
errorMessage: 'No health check result available',
};
}
const key = keys[0];
const serviceResult = result.value[key];
const details = this.transformServiceDetails(serviceResult.details);
const { status, message, ...detailsWithoutStatus } = serviceResult;
const indicator = HEALTH_INDICATORS[indicatorId];
const transformedDetails =
this.transformServiceDetails(detailsWithoutStatus);
return {
id: indicatorId,
label: indicator.label,
description: indicator.description,
status: this.transformStatus(serviceResult.status),
details: details ? JSON.stringify(details) : undefined,
status: this.transformStatus(status),
errorMessage: message,
details:
Object.keys(transformedDetails).length > 0
? JSON.stringify(transformedDetails)
: undefined,
queues: serviceResult.queues,
};
}
@ -83,7 +104,10 @@ export class AdminPanelHealthService {
return {
...HEALTH_INDICATORS[indicatorId],
status: AdminPanelHealthServiceStatus.OUTAGE,
details: result.reason?.message?.toString(),
errorMessage: result.reason?.message?.toString(),
details: result.reason?.details
? JSON.stringify(result.reason.details)
: undefined,
};
}
@ -117,13 +141,19 @@ export class AdminPanelHealthService {
}
async getSystemHealthStatus(): Promise<SystemHealth> {
const [databaseResult, redisResult, workerResult, accountSyncResult] =
await Promise.allSettled([
this.databaseHealth.isHealthy(),
this.redisHealth.isHealthy(),
this.workerHealth.isHealthy(),
this.connectedAccountHealth.isHealthy(),
]);
const [
databaseResult,
redisResult,
workerResult,
accountSyncResult,
appResult,
] = await Promise.allSettled([
this.databaseHealth.isHealthy(),
this.redisHealth.isHealthy(),
this.workerHealth.isHealthy(),
this.connectedAccountHealth.isHealthy(),
this.appHealth.isHealthy(),
]);
return {
services: [
@ -151,6 +181,11 @@ export class AdminPanelHealthService {
HealthIndicatorId.connectedAccount,
).status,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.app],
status: this.getServiceStatus(appResult, HealthIndicatorId.app)
.status,
},
],
};
}

View File

@ -28,4 +28,9 @@ export const HEALTH_INDICATORS: Record<HealthIndicatorId, HealthIndicatorInfo> =
label: 'Connected Account Status',
description: 'Connected accounts status',
},
[HealthIndicatorId.app]: {
id: HealthIndicatorId.app,
label: 'App Status',
description: 'Workspace metadata migration status check',
},
};

View File

@ -17,6 +17,8 @@ export class AdminPanelHealthServiceData {
@Field(() => AdminPanelHealthServiceStatus)
status: AdminPanelHealthServiceStatus;
@Field(() => String, { nullable: true })
errorMessage?: string;
@Field(() => String, { nullable: true })
details?: string;

View File

@ -1,8 +1,8 @@
import { registerEnumType } from '@nestjs/graphql';
export enum AdminPanelHealthServiceStatus {
OPERATIONAL = 'operational',
OUTAGE = 'outage',
OPERATIONAL = 'OPERATIONAL',
OUTAGE = 'OUTAGE',
}
registerEnumType(AdminPanelHealthServiceStatus, {

View File

@ -12,4 +12,5 @@ export const HEALTH_ERROR_MESSAGES = {
CALENDAR_SYNC_TIMEOUT: 'Calendar sync check timeout',
CALENDAR_SYNC_CHECK_FAILED: 'Calendar sync check failed',
CALENDAR_SYNC_HIGH_FAILURE_RATE: 'High failure rate in calendar sync jobs',
APP_HEALTH_CHECK_FAILED: 'App health check failed',
} as const;

View File

@ -2,6 +2,7 @@ import { HealthCheckService } from '@nestjs/terminus';
import { Test, TestingModule } from '@nestjs/testing';
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
@ -34,6 +35,10 @@ describe('HealthController', () => {
provide: ConnectedAccountHealth,
useValue: { isHealthy: jest.fn() },
},
{
provide: AppHealthIndicator,
useValue: { isHealthy: jest.fn() },
},
],
}).compile();

View File

@ -2,11 +2,11 @@ import { BadRequestException, Controller, Get, Param } from '@nestjs/common';
import { HealthCheck, HealthCheckService } from '@nestjs/terminus';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
@Controller('healthz')
export class HealthController {
constructor(
@ -15,6 +15,7 @@ export class HealthController {
private readonly redisHealth: RedisHealthIndicator,
private readonly workerHealth: WorkerHealthIndicator,
private readonly connectedAccountHealth: ConnectedAccountHealth,
private readonly appHealth: AppHealthIndicator,
) {}
@Get()
@ -32,6 +33,7 @@ export class HealthController {
[HealthIndicatorId.worker]: () => this.workerHealth.isHealthy(),
[HealthIndicatorId.connectedAccount]: () =>
this.connectedAccountHealth.isHealthy(),
[HealthIndicatorId.app]: () => this.appHealth.isHealthy(),
};
if (!(indicatorId in checks)) {

View File

@ -5,6 +5,7 @@ export enum HealthIndicatorId {
redis = 'redis',
worker = 'worker',
connectedAccount = 'connectedAccount',
app = 'app',
}
registerEnumType(HealthIndicatorId, {

View File

@ -3,7 +3,11 @@ import { TerminusModule } from '@nestjs/terminus';
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { HealthCacheService } from './health-cache.service';
@ -12,7 +16,13 @@ import { DatabaseHealthIndicator } from './indicators/database.health';
import { RedisHealthIndicator } from './indicators/redis.health';
import { WorkerHealthIndicator } from './indicators/worker.health';
@Module({
imports: [TerminusModule, RedisClientModule],
imports: [
TerminusModule,
RedisClientModule,
WorkspaceHealthModule,
ObjectMetadataModule,
WorkspaceMigrationModule,
],
controllers: [HealthController, MetricsController],
providers: [
HealthCacheService,
@ -20,6 +30,7 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
RedisHealthIndicator,
WorkerHealthIndicator,
ConnectedAccountHealth,
AppHealthIndicator,
],
exports: [
HealthCacheService,
@ -27,6 +38,7 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
RedisHealthIndicator,
WorkerHealthIndicator,
ConnectedAccountHealth,
AppHealthIndicator,
],
})
export class HealthModule {}

View File

@ -0,0 +1,167 @@
import { HealthIndicatorService } from '@nestjs/terminus';
import { Test, TestingModule } from '@nestjs/testing';
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
describe('AppHealthIndicator', () => {
let service: AppHealthIndicator;
let objectMetadataService: jest.Mocked<ObjectMetadataService>;
let workspaceHealthService: jest.Mocked<WorkspaceHealthService>;
let workspaceMigrationService: jest.Mocked<WorkspaceMigrationService>;
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
beforeEach(async () => {
objectMetadataService = {
findMany: jest.fn(),
} as any;
workspaceHealthService = {
healthCheck: jest.fn(),
} as any;
workspaceMigrationService = {
getPendingMigrations: jest.fn(),
} as any;
healthIndicatorService = {
check: jest.fn().mockReturnValue({
up: jest.fn().mockImplementation((data) => ({
app: { status: 'up', ...data },
})),
down: jest.fn().mockImplementation((data) => ({
app: { status: 'down', ...data },
})),
}),
} as any;
const module: TestingModule = await Test.createTestingModule({
providers: [
AppHealthIndicator,
{
provide: ObjectMetadataService,
useValue: objectMetadataService,
},
{
provide: WorkspaceHealthService,
useValue: workspaceHealthService,
},
{
provide: WorkspaceMigrationService,
useValue: workspaceMigrationService,
},
{
provide: HealthIndicatorService,
useValue: healthIndicatorService,
},
],
}).compile();
service = module.get<AppHealthIndicator>(AppHealthIndicator);
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
it('should return up status when no issues and no pending migrations', async () => {
objectMetadataService.findMany.mockResolvedValue([
{
id: '1',
workspaceId: 'workspace1',
} as any,
{
id: '2',
workspaceId: 'workspace2',
} as any,
]);
workspaceMigrationService.getPendingMigrations.mockResolvedValue([]);
const result = await service.isHealthy();
expect(result.app.status).toBe('up');
expect(result.app.details.overview.totalWorkspacesCount).toBe(2);
expect(result.app.details.overview.criticalWorkspacesCount).toBe(0);
expect(result.app.details.criticalWorkspaces).toBe(null);
expect(result.app.details.system.nodeVersion).toBeDefined();
expect(result.app.details.system.timestamp).toBeDefined();
});
it('should return down status when there are pending migrations', async () => {
objectMetadataService.findMany.mockResolvedValue([
{
id: '1',
workspaceId: 'workspace1',
} as any,
]);
workspaceMigrationService.getPendingMigrations.mockResolvedValue([
{
id: '1',
createdAt: new Date(),
migrations: [],
name: 'migration1',
isCustom: false,
workspaceId: 'workspace1',
} as any,
]);
const result = await service.isHealthy();
expect(result.app.status).toBe('down');
expect(result.app.details.overview.criticalWorkspacesCount).toBe(1);
expect(result.app.details.criticalWorkspaces).toEqual([
{
workspaceId: 'workspace1',
pendingMigrations: 1,
},
]);
});
it('should handle errors gracefully and maintain state history', async () => {
objectMetadataService.findMany.mockRejectedValue(
new Error('Database connection failed'),
);
const result = await service.isHealthy();
expect(result.app.status).toBe('down');
expect(result.app.message).toBe('Database connection failed');
expect(result.app.details.system.nodeVersion).toBeDefined();
expect(result.app.details.system.timestamp).toBeDefined();
expect(result.app.details.stateHistory).toBeDefined();
});
it('should maintain state history across health checks', async () => {
// First check - healthy state
objectMetadataService.findMany.mockResolvedValue([
{
id: '1',
workspaceId: 'workspace1',
} as any,
]);
workspaceMigrationService.getPendingMigrations.mockResolvedValue([]);
await service.isHealthy();
// Second check - error state
objectMetadataService.findMany.mockRejectedValue(
new Error('Database connection failed'),
);
const result = await service.isHealthy();
expect(result.app.details.stateHistory).toBeDefined();
expect(result.app.details.stateHistory.age).toBeDefined();
expect(result.app.details.stateHistory.timestamp).toBeDefined();
expect(result.app.details.stateHistory.details).toBeDefined();
});
});

View File

@ -22,11 +22,8 @@ describe('DatabaseHealthIndicator', () => {
up: jest.fn().mockImplementation((data) => ({
database: { status: 'up', ...data },
})),
down: jest.fn().mockImplementation((error) => ({
database: {
status: 'down',
error,
},
down: jest.fn().mockImplementation((data) => ({
database: { status: 'down', ...data },
})),
}),
} as any;
@ -77,10 +74,20 @@ describe('DatabaseHealthIndicator', () => {
const result = await service.isHealthy();
expect(result.database.status).toBe('up');
expect(result.database.details).toBeDefined();
expect(result.database.details.version).toBeDefined();
expect(result.database.details.connections).toBeDefined();
expect(result.database.details.performance).toBeDefined();
expect(result.database.details.system.version).toBe('PostgreSQL 15.6');
expect(result.database.details.system.timestamp).toBeDefined();
expect(result.database.details.connections).toEqual({
active: 5,
max: 100,
utilizationPercent: 5,
});
expect(result.database.details.performance).toEqual({
cacheHitRatio: '96%',
deadlocks: 0,
slowQueries: 0,
});
expect(result.database.details.databaseSize).toBe('1 GB');
expect(result.database.details.top10Tables).toEqual([{ table_stats: [] }]);
});
it('should return down status when database fails', async () => {
@ -91,9 +98,11 @@ describe('DatabaseHealthIndicator', () => {
const result = await service.isHealthy();
expect(result.database.status).toBe('down');
expect(result.database.error).toBe(
expect(result.database.message).toBe(
HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED,
);
expect(result.database.details.system.timestamp).toBeDefined();
expect(result.database.details.stateHistory).toBeDefined();
});
it('should timeout after specified duration', async () => {
@ -111,6 +120,63 @@ describe('DatabaseHealthIndicator', () => {
const result = await healthCheckPromise;
expect(result.database.status).toBe('down');
expect(result.database.error).toBe(HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT);
expect(result.database.message).toBe(
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
);
expect(result.database.details.stateHistory).toBeDefined();
});
it('should maintain state history across health checks', async () => {
// First check - healthy state
const mockResponses = [
[{ version: 'PostgreSQL 15.6' }],
[{ count: '5' }],
[{ max_connections: '100' }],
[{ uptime: '3600' }],
[{ size: '1 GB' }],
[{ table_stats: [] }],
[{ ratio: '95.5' }],
[{ deadlocks: '0' }],
[{ count: '0' }],
];
mockResponses.forEach((response) => {
dataSource.query.mockResolvedValueOnce(response);
});
const firstResult = await service.isHealthy();
expect(firstResult.database.status).toBe('up');
// Second check - error state
dataSource.query.mockRejectedValueOnce(
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
);
const result = await service.isHealthy();
expect(result.database.details.stateHistory).toMatchObject({
age: expect.any(Number),
timestamp: expect.any(Date),
details: {
system: {
version: 'PostgreSQL 15.6',
timestamp: expect.any(String),
uptime: expect.any(String),
},
connections: {
active: 5,
max: 100,
utilizationPercent: 5,
},
performance: {
cacheHitRatio: '96%',
deadlocks: 0,
slowQueries: 0,
},
databaseSize: '1 GB',
top10Tables: [{ table_stats: [] }],
},
});
});
});

View File

@ -35,7 +35,7 @@ describe('RedisHealthIndicator', () => {
down: jest.fn().mockImplementation((error) => ({
redis: {
status: 'down',
error,
...error,
},
})),
}),
@ -72,35 +72,43 @@ describe('RedisHealthIndicator', () => {
mockRedis.info
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
.mockResolvedValueOnce(
'used_memory_human:1.2G\nused_memory_peak_human:1.5G\nmem_fragmentation_ratio:1.5\n',
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
)
.mockResolvedValueOnce('connected_clients:5\n')
.mockResolvedValueOnce('connected_clients:5\r\n')
.mockResolvedValueOnce(
'total_connections_received:100\nkeyspace_hits:90\nkeyspace_misses:10\n',
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
);
const result = await service.isHealthy();
expect(result.redis.status).toBe('up');
expect(result.redis.details).toBeDefined();
expect(result.redis.details.version).toBe('7.0.0');
expect(result.redis.details.system.version).toBe('7.0.0');
expect(result.redis.details.system.timestamp).toBeDefined();
expect(result.redis.details.memory).toEqual({
used: '1.2G',
peak: '1.5G',
fragmentation: 1.5,
});
});
it('should return down status when redis fails', async () => {
mockRedis.ping.mockRejectedValueOnce(
mockRedis.info.mockRejectedValueOnce(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
const result = await service.isHealthy();
expect(result.redis.status).toBe('down');
expect(result.redis.error).toBe(
expect(result.redis.message).toBe(
HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
);
expect(result.redis.details.system.timestamp).toBeDefined();
expect(result.redis.details.stateHistory).toBeDefined();
});
it('should timeout after specified duration', async () => {
mockRedis.ping.mockImplementationOnce(
mockRedis.info.mockImplementationOnce(
() =>
new Promise((resolve) =>
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
@ -110,24 +118,38 @@ describe('RedisHealthIndicator', () => {
const healthCheckPromise = service.isHealthy();
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
const result = await healthCheckPromise;
expect(result.redis.status).toBe('down');
expect(result.redis.error).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT);
expect(result.redis.message).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT);
expect(result.redis.details.system.timestamp).toBeDefined();
expect(result.redis.details.stateHistory).toBeDefined();
});
it('should handle partial failures in health details collection', async () => {
it('should maintain state history across health checks', async () => {
// First check - healthy state
mockRedis.info
.mockResolvedValueOnce('redis_version:7.0.0') // info
.mockResolvedValueOnce('used_memory_human:1.2G') // memory
.mockResolvedValueOnce('connected_clients:5') // clients
.mockResolvedValueOnce('total_connections_received:100'); // stats
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
.mockResolvedValueOnce(
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
)
.mockResolvedValueOnce('connected_clients:5\r\n')
.mockResolvedValueOnce(
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
);
await service.isHealthy();
// Second check - error state
mockRedis.info.mockRejectedValueOnce(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
const result = await service.isHealthy();
expect(result.redis.status).toBe('up');
expect(result.redis.details).toBeDefined();
expect(result.redis.details.version).toBe('7.0.0');
expect(result.redis.details.stateHistory).toBeDefined();
expect(result.redis.details.stateHistory.age).toBeDefined();
expect(result.redis.details.stateHistory.timestamp).toBeDefined();
expect(result.redis.details.stateHistory.details).toBeDefined();
});
});

View File

@ -0,0 +1,96 @@
import { Injectable } from '@nestjs/common';
import {
HealthIndicatorResult,
HealthIndicatorService,
} from '@nestjs/terminus';
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
@Injectable()
export class AppHealthIndicator {
private stateManager = new HealthStateManager();
constructor(
private readonly healthIndicatorService: HealthIndicatorService,
private readonly workspaceHealthService: WorkspaceHealthService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
) {}
async isHealthy(): Promise<HealthIndicatorResult> {
const indicator = this.healthIndicatorService.check('app');
try {
const workspaces = await this.objectMetadataService.findMany();
const workspaceIds = [...new Set(workspaces.map((w) => w.workspaceId))];
const workspaceStats = await Promise.all(
workspaceIds.map(async (workspaceId) => {
const pendingMigrations =
await this.workspaceMigrationService.getPendingMigrations(
workspaceId,
);
return {
workspaceId,
pendingMigrations: pendingMigrations.length,
isCritical: pendingMigrations.length > 0,
};
}),
);
const details = {
system: {
nodeVersion: process.version,
timestamp: new Date().toISOString(),
},
overview: {
totalWorkspacesCount: workspaceIds.length,
criticalWorkspacesCount: workspaceStats.filter(
(stat) => stat.isCritical,
).length,
},
criticalWorkspaces:
workspaceStats.filter((stat) => stat.isCritical).length > 0
? workspaceStats
.filter((stat) => stat.isCritical)
.map((stat) => ({
workspaceId: stat.workspaceId,
pendingMigrations: stat.pendingMigrations,
}))
: null,
};
const isHealthy = workspaceStats.every((stat) => !stat.isCritical);
if (isHealthy) {
this.stateManager.updateState(details);
return indicator.up({ details });
}
this.stateManager.updateState(details);
return indicator.down({
message: `Found ${details.criticalWorkspaces?.length} workspaces with pending migrations`,
details,
});
} catch (error) {
const stateWithAge = this.stateManager.getStateWithAge();
return indicator.down({
message: error.message,
details: {
system: {
nodeVersion: process.version,
timestamp: new Date().toISOString(),
},
stateHistory: stateWithAge,
},
});
}
}
}

View File

@ -9,9 +9,12 @@ import { DataSource } from 'typeorm';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
@Injectable()
export class DatabaseHealthIndicator {
private stateManager = new HealthStateManager();
constructor(
@InjectDataSource('core')
private readonly dataSource: DataSource,
@ -69,35 +72,50 @@ export class DatabaseHealthIndicator {
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
);
return indicator.up({
details: {
const details = {
system: {
timestamp: new Date().toISOString(),
version: versionResult.version,
connections: {
active: parseInt(activeConnections.count),
max: parseInt(maxConnections.max_connections),
utilizationPercent: Math.round(
(parseInt(activeConnections.count) /
parseInt(maxConnections.max_connections)) *
100,
),
},
uptime: Math.round(uptime.uptime / 3600) + ' hours',
databaseSize: databaseSize.size,
performance: {
cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%',
deadlocks: parseInt(deadlocks.deadlocks),
slowQueries: parseInt(slowQueries.count),
},
top10Tables: tableStats,
},
});
connections: {
active: parseInt(activeConnections.count),
max: parseInt(maxConnections.max_connections),
utilizationPercent: Math.round(
(parseInt(activeConnections.count) /
parseInt(maxConnections.max_connections)) *
100,
),
},
databaseSize: databaseSize.size,
performance: {
cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%',
deadlocks: parseInt(deadlocks.deadlocks),
slowQueries: parseInt(slowQueries.count),
},
top10Tables: tableStats,
};
this.stateManager.updateState(details);
return indicator.up({ details });
} catch (error) {
const errorMessage =
const message =
error.message === HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
? HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
: HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED;
return indicator.down(errorMessage);
const stateWithAge = this.stateManager.getStateWithAge();
return indicator.down({
message,
details: {
system: {
timestamp: new Date().toISOString(),
},
stateHistory: stateWithAge,
},
});
}
}
}

View File

@ -6,10 +6,13 @@ import {
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util';
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
@Injectable()
export class RedisHealthIndicator {
private stateManager = new HealthStateManager();
constructor(
private readonly redisClient: RedisClientService,
private readonly healthIndicatorService: HealthIndicatorService,
@ -48,47 +51,62 @@ export class RedisHealthIndicator {
const clientsData = parseInfo(clients);
const statsData = parseInfo(stats);
return indicator.up({
details: {
const details = {
system: {
timestamp: new Date().toISOString(),
version: infoData.redis_version,
uptime:
Math.round(parseInt(infoData.uptime_in_seconds) / 3600) + ' hours',
memory: {
used: memoryData.used_memory_human,
peak: memoryData.used_memory_peak_human,
fragmentation: parseFloat(memoryData.mem_fragmentation_ratio),
},
connections: {
current: parseInt(clientsData.connected_clients),
total: parseInt(statsData.total_connections_received),
rejected: parseInt(statsData.rejected_connections),
},
performance: {
opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec),
hitRate: statsData.keyspace_hits
? Math.round(
(parseInt(statsData.keyspace_hits) /
(parseInt(statsData.keyspace_hits) +
parseInt(statsData.keyspace_misses))) *
100,
) + '%'
: '0%',
evictedKeys: parseInt(statsData.evicted_keys),
expiredKeys: parseInt(statsData.expired_keys),
},
replication: {
role: infoData.role,
connectedSlaves: parseInt(infoData.connected_slaves || '0'),
},
},
});
memory: {
used: memoryData.used_memory_human,
peak: memoryData.used_memory_peak_human,
fragmentation: parseFloat(memoryData.mem_fragmentation_ratio),
},
connections: {
current: parseInt(clientsData.connected_clients),
total: parseInt(statsData.total_connections_received),
rejected: parseInt(statsData.rejected_connections),
},
performance: {
opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec),
hitRate: statsData.keyspace_hits
? Math.round(
(parseInt(statsData.keyspace_hits) /
(parseInt(statsData.keyspace_hits) +
parseInt(statsData.keyspace_misses))) *
100,
) + '%'
: '0%',
evictedKeys: parseInt(statsData.evicted_keys),
expiredKeys: parseInt(statsData.expired_keys),
},
replication: {
role: infoData.role,
connectedSlaves: parseInt(infoData.connected_slaves || '0'),
},
};
this.stateManager.updateState(details);
return indicator.up({ details });
} catch (error) {
const errorMessage =
const message =
error.message === HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
? HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED;
return indicator.down(errorMessage);
const stateWithAge = this.stateManager.getStateWithAge();
return indicator.down({
message,
details: {
system: {
timestamp: new Date().toISOString(),
},
stateHistory: stateWithAge,
},
});
}
}
}

View File

@ -0,0 +1,22 @@
export class HealthStateManager {
private lastKnownState: {
timestamp: Date;
details: Record<string, any>;
} | null = null;
updateState(details: Record<string, any>) {
this.lastKnownState = {
timestamp: new Date(),
details,
};
}
getStateWithAge() {
return this.lastKnownState
? {
...this.lastKnownState,
age: Date.now() - this.lastKnownState.timestamp.getTime(),
}
: 'No previous state available';
}
}