Health monitor status for admin panel (#10186)
# Health Monitoring for Self-Hosted Instances
This PR implements basic health monitoring for self-hosted instances in
the admin panel.
## Service Status Checks
We're adding real-time health checks for:
- Redis Connection
- Database Connection
- Worker Status
- Message Sync Status
## Existing Functionality
We already have message sync and captcha counters that store aggregated
metrics in cache within a configurable time window (default: 5 minutes).
## New Endpoints
1. `/healthz` - Basic server health check for Kubernetes pod monitoring
2. `/healthz/{serviceName}` - Individual service health checks (returns
200 if healthy)
3. `/metricsz/{metricName}` - Time-windowed metrics (message sync,
captcha)
4. GraphQL resolver in admin panel for UI consumption
All endpoints use the same underlying service, with different
presentation layers for infrastructure and UI needs.
---------
Co-authored-by: Félix Malfait <felix@twenty.com>
This commit is contained in:
nitin
@ -248,6 +248,14 @@ const SettingsAdminContent = lazy(() =>
|
||||
),
|
||||
);
|
||||
|
||||
const SettingsAdminIndicatorHealthStatus = lazy(() =>
|
||||
import(
|
||||
'~/pages/settings/admin-panel/SettingsAdminIndicatorHealthStatus'
|
||||
).then((module) => ({
|
||||
default: module.SettingsAdminIndicatorHealthStatus,
|
||||
})),
|
||||
);
|
||||
|
||||
const SettingsLab = lazy(() =>
|
||||
import('~/pages/settings/lab/SettingsLab').then((module) => ({
|
||||
default: module.SettingsLab,
|
||||
@ -407,6 +415,10 @@ export const SettingsRoutes = ({
|
||||
path={SettingsPath.FeatureFlags}
|
||||
element={<SettingsAdminContent />}
|
||||
/>
|
||||
<Route
|
||||
path={SettingsPath.AdminPanelIndicatorHealthStatus}
|
||||
element={<SettingsAdminIndicatorHealthStatus />}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
<Route path={SettingsPath.Lab} element={<SettingsLab />} />
|
||||
|
||||
@ -3,7 +3,7 @@ import { SETTINGS_ADMIN_TABS } from '@/settings/admin-panel/constants/SettingsAd
|
||||
import { SETTINGS_ADMIN_TABS_ID } from '@/settings/admin-panel/constants/SettingsAdminTabsId';
|
||||
import { TabList } from '@/ui/layout/tab/components/TabList';
|
||||
import styled from '@emotion/styled';
|
||||
import { IconSettings2, IconVariable } from 'twenty-ui';
|
||||
import { IconHeart, IconSettings2, IconVariable } from 'twenty-ui';
|
||||
|
||||
const StyledTabListContainer = styled.div`
|
||||
align-items: center;
|
||||
@ -25,6 +25,11 @@ export const SettingsAdminContent = () => {
|
||||
title: 'Env Variables',
|
||||
Icon: IconVariable,
|
||||
},
|
||||
{
|
||||
id: SETTINGS_ADMIN_TABS.HEALTH_STATUS,
|
||||
title: 'Health Status',
|
||||
Icon: IconHeart,
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
|
||||
@ -15,6 +15,7 @@ import { useRecoilState, useRecoilValue } from 'recoil';
|
||||
import { getImageAbsoluteURI, isDefined } from 'twenty-shared';
|
||||
import {
|
||||
Button,
|
||||
GithubVersionLink,
|
||||
H1Title,
|
||||
H1TitleFontColor,
|
||||
H2Title,
|
||||
@ -24,6 +25,8 @@ import {
|
||||
import { REACT_APP_SERVER_BASE_URL } from '~/config';
|
||||
import { useUserLookupAdminPanelMutation } from '~/generated/graphql';
|
||||
|
||||
import packageJson from '../../../../../package.json';
|
||||
|
||||
const StyledLinkContainer = styled.div`
|
||||
margin-right: ${({ theme }) => theme.spacing(2)};
|
||||
width: 100%;
|
||||
@ -120,6 +123,11 @@ export const SettingsAdminGeneral = () => {
|
||||
|
||||
return (
|
||||
<>
|
||||
<Section>
|
||||
<H2Title title="About" description="Version of the application" />
|
||||
<GithubVersionLink version={packageJson.version} />
|
||||
</Section>
|
||||
|
||||
<Section>
|
||||
<H2Title
|
||||
title={
|
||||
@ -176,6 +184,7 @@ export const SettingsAdminGeneral = () => {
|
||||
behaveAsLinks={false}
|
||||
/>
|
||||
</StyledTabListContainer>
|
||||
|
||||
<StyledContentContainer>
|
||||
<SettingsAdminWorkspaceContent activeWorkspace={activeWorkspace} />
|
||||
</StyledContentContainer>
|
||||
|
||||
@ -0,0 +1,44 @@
|
||||
import { Table } from '@/ui/layout/table/components/Table';
|
||||
import { TableCell } from '@/ui/layout/table/components/TableCell';
|
||||
import { TableHeader } from '@/ui/layout/table/components/TableHeader';
|
||||
import { TableRow } from '@/ui/layout/table/components/TableRow';
|
||||
|
||||
export const SettingsAdminHealthMessageSyncCountersTable = ({
|
||||
details,
|
||||
}: {
|
||||
details: string | null | undefined;
|
||||
}) => {
|
||||
const parsedDetails = details ? JSON.parse(details) : null;
|
||||
if (!parsedDetails) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Table>
|
||||
<TableRow>
|
||||
<TableHeader>Status</TableHeader>
|
||||
<TableHeader align="right">Count</TableHeader>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell>Message Not Synced</TableCell>
|
||||
<TableCell align="right">{parsedDetails.counters.NOT_SYNCED}</TableCell>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell>Message Sync Ongoing</TableCell>
|
||||
<TableCell align="right">{parsedDetails.counters.ONGOING}</TableCell>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell>Total Jobs</TableCell>
|
||||
<TableCell align="right">{parsedDetails.totalJobs}</TableCell>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell>Failed Jobs</TableCell>
|
||||
<TableCell align="right">{parsedDetails.failedJobs}</TableCell>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell>Failure Rate</TableCell>
|
||||
<TableCell align="right">{parsedDetails.failureRate}%</TableCell>
|
||||
</TableRow>
|
||||
</Table>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,66 @@
|
||||
import { SettingsAdminHealthMessageSyncCountersTable } from '@/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable';
|
||||
import { SettingsHealthStatusListCard } from '@/settings/admin-panel/components/SettingsHealthStatusListCard';
|
||||
import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService';
|
||||
import styled from '@emotion/styled';
|
||||
import { H2Title, Section } from 'twenty-ui';
|
||||
import {
|
||||
AdminPanelHealthServiceStatus,
|
||||
useGetSystemHealthStatusQuery,
|
||||
} from '~/generated/graphql';
|
||||
|
||||
const StyledErrorMessage = styled.div`
|
||||
color: ${({ theme }) => theme.color.red};
|
||||
margin-top: ${({ theme }) => theme.spacing(2)};
|
||||
`;
|
||||
|
||||
export const SettingsAdminHealthStatus = () => {
|
||||
const { data, loading } = useGetSystemHealthStatusQuery({
|
||||
fetchPolicy: 'network-only',
|
||||
});
|
||||
|
||||
const services = [
|
||||
{
|
||||
id: 'DATABASE',
|
||||
name: 'Database Status',
|
||||
...data?.getSystemHealthStatus.database,
|
||||
},
|
||||
{ id: 'REDIS', name: 'Redis Status', ...data?.getSystemHealthStatus.redis },
|
||||
{
|
||||
id: 'WORKER',
|
||||
name: 'Worker Status',
|
||||
status: data?.getSystemHealthStatus.worker.status,
|
||||
queues: data?.getSystemHealthStatus.worker.queues,
|
||||
},
|
||||
].filter((service): service is AdminHealthService => !!service.status);
|
||||
|
||||
const isMessageSyncCounterDown =
|
||||
!data?.getSystemHealthStatus.messageSync.status ||
|
||||
data?.getSystemHealthStatus.messageSync.status ===
|
||||
AdminPanelHealthServiceStatus.OUTAGE;
|
||||
|
||||
return (
|
||||
<>
|
||||
<Section>
|
||||
<H2Title title="Health Status" description="How your system is doing" />
|
||||
<SettingsHealthStatusListCard services={services} loading={loading} />
|
||||
</Section>
|
||||
|
||||
<Section>
|
||||
<H2Title
|
||||
title="Message Sync Status"
|
||||
description="How your message sync is doing"
|
||||
/>
|
||||
{isMessageSyncCounterDown ? (
|
||||
<StyledErrorMessage>
|
||||
{data?.getSystemHealthStatus.messageSync.details ||
|
||||
'Message sync status is unavailable'}
|
||||
</StyledErrorMessage>
|
||||
) : (
|
||||
<SettingsAdminHealthMessageSyncCountersTable
|
||||
details={data?.getSystemHealthStatus.messageSync.details}
|
||||
/>
|
||||
)}
|
||||
</Section>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,27 @@
|
||||
import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService';
|
||||
import styled from '@emotion/styled';
|
||||
import { Status } from 'twenty-ui';
|
||||
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
|
||||
|
||||
const StyledRowRightContainer = styled.div`
|
||||
align-items: center;
|
||||
display: flex;
|
||||
gap: ${({ theme }) => theme.spacing(1)};
|
||||
`;
|
||||
|
||||
export const SettingsAdminHealthStatusRightContainer = ({
|
||||
service,
|
||||
}: {
|
||||
service: AdminHealthService;
|
||||
}) => {
|
||||
return (
|
||||
<StyledRowRightContainer>
|
||||
{service.status === AdminPanelHealthServiceStatus.OPERATIONAL && (
|
||||
<Status color="green" text="Operational" weight="medium" />
|
||||
)}
|
||||
{service.status === AdminPanelHealthServiceStatus.OUTAGE && (
|
||||
<Status color="red" text="Outage" weight="medium" />
|
||||
)}
|
||||
</StyledRowRightContainer>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,103 @@
|
||||
import { SettingsListCard } from '@/settings/components/SettingsListCard';
|
||||
import { Table } from '@/ui/layout/table/components/Table';
|
||||
import { TableCell } from '@/ui/layout/table/components/TableCell';
|
||||
import { TableRow } from '@/ui/layout/table/components/TableRow';
|
||||
import styled from '@emotion/styled';
|
||||
import { AnimatedExpandableContainer, Status } from 'twenty-ui';
|
||||
import {
|
||||
AdminPanelHealthServiceStatus,
|
||||
AdminPanelWorkerQueueHealth,
|
||||
} from '~/generated/graphql';
|
||||
|
||||
const StyledExpandedContent = styled.div`
|
||||
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
padding-top: ${({ theme }) => theme.spacing(1)};
|
||||
padding-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||
padding-right: ${({ theme }) => theme.spacing(3)};
|
||||
`;
|
||||
|
||||
const StyledContainer = styled.div`
|
||||
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
margin-top: ${({ theme }) => theme.spacing(5)};
|
||||
`;
|
||||
|
||||
const StyledTableRow = styled(TableRow)`
|
||||
height: ${({ theme }) => theme.spacing(6)};
|
||||
`;
|
||||
|
||||
const StyledQueueMetricsTitle = styled.div`
|
||||
color: ${({ theme }) => theme.font.color.primary};
|
||||
font-size: ${({ theme }) => theme.font.size.sm};
|
||||
font-weight: ${({ theme }) => theme.font.weight.medium};
|
||||
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||
`;
|
||||
|
||||
export const SettingsAdminQueueExpandableContainer = ({
|
||||
queues,
|
||||
selectedQueue,
|
||||
}: {
|
||||
queues: AdminPanelWorkerQueueHealth[];
|
||||
selectedQueue: string | null;
|
||||
}) => {
|
||||
const selectedQueueData = queues.find(
|
||||
(queue) => queue.name === selectedQueue,
|
||||
);
|
||||
|
||||
return (
|
||||
<AnimatedExpandableContainer
|
||||
isExpanded={!!selectedQueue}
|
||||
mode="fit-content"
|
||||
>
|
||||
{selectedQueueData && (
|
||||
<>
|
||||
<StyledContainer>
|
||||
<SettingsListCard
|
||||
items={[{ ...selectedQueueData, id: selectedQueueData.name }]}
|
||||
getItemLabel={(
|
||||
item: AdminPanelWorkerQueueHealth & { id: string },
|
||||
) => item.name}
|
||||
isLoading={false}
|
||||
RowRightComponent={({
|
||||
item,
|
||||
}: {
|
||||
item: AdminPanelWorkerQueueHealth;
|
||||
}) => (
|
||||
<Status
|
||||
color={
|
||||
item.status === AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
? 'green'
|
||||
: 'red'
|
||||
}
|
||||
text={item.status.toLowerCase()}
|
||||
weight="medium"
|
||||
/>
|
||||
)}
|
||||
/>
|
||||
</StyledContainer>
|
||||
<StyledQueueMetricsTitle> Metrics:</StyledQueueMetricsTitle>
|
||||
<StyledExpandedContent>
|
||||
<Table>
|
||||
<StyledTableRow>
|
||||
<TableCell align="left">Workers</TableCell>
|
||||
<TableCell align="right">{selectedQueueData.workers}</TableCell>
|
||||
</StyledTableRow>
|
||||
{Object.entries(selectedQueueData.metrics)
|
||||
.filter(([key]) => key !== '__typename')
|
||||
.map(([key, value]) => (
|
||||
<StyledTableRow key={key}>
|
||||
<TableCell align="left">
|
||||
{key.charAt(0).toUpperCase() + key.slice(1)}
|
||||
</TableCell>
|
||||
<TableCell align="right">{value}</TableCell>
|
||||
</StyledTableRow>
|
||||
))}
|
||||
</Table>
|
||||
</StyledExpandedContent>
|
||||
</>
|
||||
)}
|
||||
</AnimatedExpandableContainer>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,52 @@
|
||||
import styled from '@emotion/styled';
|
||||
import { Button } from 'twenty-ui';
|
||||
import {
|
||||
AdminPanelHealthServiceStatus,
|
||||
AdminPanelWorkerQueueHealth,
|
||||
} from '~/generated/graphql';
|
||||
|
||||
const StyledQueueButtonsRow = styled.div`
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: ${({ theme }) => theme.spacing(2)};
|
||||
margin-top: ${({ theme }) => theme.spacing(6)};
|
||||
`;
|
||||
|
||||
const StyledQueueHealthButton = styled(Button)<{
|
||||
isSelected?: boolean;
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
}>`
|
||||
${({ isSelected, theme, status }) =>
|
||||
isSelected &&
|
||||
`
|
||||
background-color: ${
|
||||
status === AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
? theme.tag.background.green
|
||||
: theme.tag.background.red
|
||||
};
|
||||
`}
|
||||
`;
|
||||
export const SettingsAdminQueueHealthButtons = ({
|
||||
queues,
|
||||
selectedQueue,
|
||||
toggleQueueVisibility,
|
||||
}: {
|
||||
queues: AdminPanelWorkerQueueHealth[];
|
||||
selectedQueue: string | null;
|
||||
toggleQueueVisibility: (queueName: string) => void;
|
||||
}) => {
|
||||
return (
|
||||
<StyledQueueButtonsRow>
|
||||
{queues.map((queue) => (
|
||||
<StyledQueueHealthButton
|
||||
key={queue.name}
|
||||
onClick={() => toggleQueueVisibility(queue.name)}
|
||||
title={queue.name}
|
||||
variant="secondary"
|
||||
isSelected={selectedQueue === queue.name}
|
||||
status={queue.status}
|
||||
/>
|
||||
))}
|
||||
</StyledQueueButtonsRow>
|
||||
);
|
||||
};
|
||||
@ -1,5 +1,6 @@
|
||||
import { SettingsAdminEnvVariables } from '@/settings/admin-panel/components/SettingsAdminEnvVariables';
|
||||
import { SettingsAdminGeneral } from '@/settings/admin-panel/components/SettingsAdminGeneral';
|
||||
import { SettingsAdminHealthStatus } from '@/settings/admin-panel/components/SettingsAdminHealthStatus';
|
||||
import { SETTINGS_ADMIN_TABS } from '@/settings/admin-panel/constants/SettingsAdminTabs';
|
||||
import { SETTINGS_ADMIN_TABS_ID } from '@/settings/admin-panel/constants/SettingsAdminTabsId';
|
||||
import { useTabList } from '@/ui/layout/tab/hooks/useTabList';
|
||||
@ -12,6 +13,8 @@ export const SettingsAdminTabContent = () => {
|
||||
return <SettingsAdminGeneral />;
|
||||
case SETTINGS_ADMIN_TABS.ENV_VARIABLES:
|
||||
return <SettingsAdminEnvVariables />;
|
||||
case SETTINGS_ADMIN_TABS.HEALTH_STATUS:
|
||||
return <SettingsAdminHealthStatus />;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService';
|
||||
import styled from '@emotion/styled';
|
||||
|
||||
import { SettingsPath } from '@/types/SettingsPath';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { getSettingsPath } from '~/utils/navigation/getSettingsPath';
|
||||
import { SettingsListCard } from '../../components/SettingsListCard';
|
||||
import { SettingsAdminHealthStatusRightContainer } from './SettingsAdminHealthStatusRightContainer';
|
||||
|
||||
const StyledLink = styled(Link)`
|
||||
text-decoration: none;
|
||||
`;
|
||||
|
||||
export const SettingsHealthStatusListCard = ({
|
||||
services,
|
||||
loading,
|
||||
}: {
|
||||
services: Array<AdminHealthService>;
|
||||
loading?: boolean;
|
||||
}) => {
|
||||
return (
|
||||
<>
|
||||
{services.map((service) => (
|
||||
<>
|
||||
<StyledLink
|
||||
to={getSettingsPath(SettingsPath.AdminPanelIndicatorHealthStatus, {
|
||||
indicatorName: service.id,
|
||||
})}
|
||||
>
|
||||
<SettingsListCard
|
||||
items={[service]}
|
||||
getItemLabel={(service) => service.name}
|
||||
isLoading={loading}
|
||||
RowRightComponent={({ item: service }) => (
|
||||
<SettingsAdminHealthStatusRightContainer service={service} />
|
||||
)}
|
||||
/>
|
||||
</StyledLink>
|
||||
</>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
@ -1,4 +1,5 @@
|
||||
export const SETTINGS_ADMIN_TABS = {
|
||||
GENERAL: 'general',
|
||||
ENV_VARIABLES: 'env-variables',
|
||||
HEALTH_STATUS: 'health-status',
|
||||
};
|
||||
|
||||
@ -0,0 +1,25 @@
|
||||
import { gql } from '@apollo/client';
|
||||
|
||||
export const GET_INDICATOR_HEALTH_STATUS = gql`
|
||||
query GetIndicatorHealthStatus(
|
||||
$indicatorName: AdminPanelIndicatorHealthStatusInputEnum!
|
||||
) {
|
||||
getIndicatorHealthStatus(indicatorName: $indicatorName) {
|
||||
status
|
||||
details
|
||||
queues {
|
||||
name
|
||||
status
|
||||
workers
|
||||
metrics {
|
||||
failed
|
||||
completed
|
||||
waiting
|
||||
active
|
||||
delayed
|
||||
prioritized
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
@ -0,0 +1,36 @@
|
||||
import { gql } from '@apollo/client';
|
||||
|
||||
export const GET_SYSTEM_HEALTH_STATUS = gql`
|
||||
query GetSystemHealthStatus {
|
||||
getSystemHealthStatus {
|
||||
database {
|
||||
status
|
||||
details
|
||||
}
|
||||
redis {
|
||||
status
|
||||
details
|
||||
}
|
||||
worker {
|
||||
status
|
||||
queues {
|
||||
name
|
||||
workers
|
||||
status
|
||||
metrics {
|
||||
failed
|
||||
completed
|
||||
waiting
|
||||
active
|
||||
delayed
|
||||
prioritized
|
||||
}
|
||||
}
|
||||
}
|
||||
messageSync {
|
||||
status
|
||||
details
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
@ -0,0 +1,12 @@
|
||||
import { useGetSystemHealthStatusQuery } from '~/generated/graphql';
|
||||
|
||||
export const useGetUptoDateHealthStatus = () => {
|
||||
const { data, loading } = useGetSystemHealthStatusQuery({
|
||||
fetchPolicy: 'network-only',
|
||||
});
|
||||
|
||||
return {
|
||||
healthStatus: data?.getSystemHealthStatus,
|
||||
healthStatusLoading: loading,
|
||||
};
|
||||
};
|
||||
@ -0,0 +1,12 @@
|
||||
import {
|
||||
AdminPanelHealthServiceData,
|
||||
AdminPanelWorkerQueueHealth,
|
||||
} from '~/generated/graphql';
|
||||
|
||||
type AdminWorkerService = AdminPanelHealthServiceData & {
|
||||
id: string;
|
||||
name: string;
|
||||
queues: AdminPanelWorkerQueueHealth[] | null | undefined;
|
||||
};
|
||||
|
||||
export type AdminHealthService = AdminWorkerService;
|
||||
@ -34,6 +34,8 @@ export enum SettingsPath {
|
||||
Releases = 'releases',
|
||||
AdminPanel = 'admin-panel',
|
||||
FeatureFlags = 'admin-panel/feature-flags',
|
||||
AdminPanelHealthStatus = 'admin-panel#health-status',
|
||||
AdminPanelIndicatorHealthStatus = 'admin-panel/health-status/:indicatorName',
|
||||
Lab = 'lab',
|
||||
Roles = 'roles',
|
||||
RoleDetail = 'roles/:roleId',
|
||||
|
||||
Reference in New Issue
Block a user