Health status worker metrics improvements (#10442)
Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
nitin
@ -43,10 +43,8 @@ export enum AdminPanelHealthServiceStatus {
|
||||
export type AdminPanelWorkerQueueHealth = {
|
||||
__typename?: 'AdminPanelWorkerQueueHealth';
|
||||
id: Scalars['String'];
|
||||
metrics: WorkerQueueMetrics;
|
||||
queueName: Scalars['String'];
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
workers: Scalars['Float'];
|
||||
};
|
||||
|
||||
export type Analytics = {
|
||||
@ -1310,6 +1308,7 @@ export type Query = {
|
||||
getPostgresCredentials?: Maybe<PostgresCredentials>;
|
||||
getProductPrices: BillingProductPricesOutput;
|
||||
getPublicWorkspaceDataByDomain: PublicWorkspaceDataOutput;
|
||||
getQueueMetrics: QueueMetricsData;
|
||||
getRoles: Array<Role>;
|
||||
getSSOIdentityProviders: Array<FindAvailableSsoidpOutput>;
|
||||
getServerlessFunctionSourceCode?: Maybe<Scalars['JSON']>;
|
||||
@ -1369,6 +1368,12 @@ export type QueryGetProductPricesArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type QueryGetQueueMetricsArgs = {
|
||||
queueName: Scalars['String'];
|
||||
timeRange?: InputMaybe<QueueMetricsTimeRange>;
|
||||
};
|
||||
|
||||
|
||||
export type QueryGetServerlessFunctionSourceCodeArgs = {
|
||||
input: GetServerlessFunctionSourceCodeInput;
|
||||
};
|
||||
@ -1413,6 +1418,35 @@ export type QueryValidatePasswordResetTokenArgs = {
|
||||
passwordResetToken: Scalars['String'];
|
||||
};
|
||||
|
||||
export type QueueMetricsData = {
|
||||
__typename?: 'QueueMetricsData';
|
||||
data: Array<QueueMetricsSeries>;
|
||||
details?: Maybe<WorkerQueueMetrics>;
|
||||
queueName: Scalars['String'];
|
||||
timeRange: QueueMetricsTimeRange;
|
||||
workers: Scalars['Float'];
|
||||
};
|
||||
|
||||
export type QueueMetricsDataPoint = {
|
||||
__typename?: 'QueueMetricsDataPoint';
|
||||
x: Scalars['Float'];
|
||||
y: Scalars['Float'];
|
||||
};
|
||||
|
||||
export type QueueMetricsSeries = {
|
||||
__typename?: 'QueueMetricsSeries';
|
||||
data: Array<QueueMetricsDataPoint>;
|
||||
id: Scalars['String'];
|
||||
};
|
||||
|
||||
export enum QueueMetricsTimeRange {
|
||||
FourHours = 'FourHours',
|
||||
OneDay = 'OneDay',
|
||||
OneHour = 'OneHour',
|
||||
SevenDays = 'SevenDays',
|
||||
TwelveHours = 'TwelveHours'
|
||||
}
|
||||
|
||||
export type Relation = {
|
||||
__typename?: 'Relation';
|
||||
sourceFieldMetadata: Field;
|
||||
@ -1970,9 +2004,11 @@ export type WorkerQueueMetrics = {
|
||||
__typename?: 'WorkerQueueMetrics';
|
||||
active: Scalars['Float'];
|
||||
completed: Scalars['Float'];
|
||||
completedData?: Maybe<Array<Scalars['Float']>>;
|
||||
delayed: Scalars['Float'];
|
||||
failed: Scalars['Float'];
|
||||
prioritized: Scalars['Float'];
|
||||
failedData?: Maybe<Array<Scalars['Float']>>;
|
||||
failureRate: Scalars['Float'];
|
||||
waiting: Scalars['Float'];
|
||||
};
|
||||
|
||||
@ -2394,7 +2430,15 @@ export type GetIndicatorHealthStatusQueryVariables = Exact<{
|
||||
}>;
|
||||
|
||||
|
||||
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus, workers: number, metrics: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, prioritized: number } }> | null } };
|
||||
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus }> | null } };
|
||||
|
||||
export type GetQueueMetricsQueryVariables = Exact<{
|
||||
queueName: Scalars['String'];
|
||||
timeRange?: InputMaybe<QueueMetricsTimeRange>;
|
||||
}>;
|
||||
|
||||
|
||||
export type GetQueueMetricsQuery = { __typename?: 'Query', getQueueMetrics: { __typename?: 'QueueMetricsData', queueName: string, timeRange: QueueMetricsTimeRange, workers: number, details?: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, failureRate: number } | null, data: Array<{ __typename?: 'QueueMetricsSeries', id: string, data: Array<{ __typename?: 'QueueMetricsDataPoint', x: number, y: number }> }> } };
|
||||
|
||||
export type GetSystemHealthStatusQueryVariables = Exact<{ [key: string]: never; }>;
|
||||
|
||||
@ -4238,15 +4282,6 @@ export const GetIndicatorHealthStatusDocument = gql`
|
||||
id
|
||||
queueName
|
||||
status
|
||||
workers
|
||||
metrics {
|
||||
failed
|
||||
completed
|
||||
waiting
|
||||
active
|
||||
delayed
|
||||
prioritized
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -4279,6 +4314,59 @@ export function useGetIndicatorHealthStatusLazyQuery(baseOptions?: Apollo.LazyQu
|
||||
export type GetIndicatorHealthStatusQueryHookResult = ReturnType<typeof useGetIndicatorHealthStatusQuery>;
|
||||
export type GetIndicatorHealthStatusLazyQueryHookResult = ReturnType<typeof useGetIndicatorHealthStatusLazyQuery>;
|
||||
export type GetIndicatorHealthStatusQueryResult = Apollo.QueryResult<GetIndicatorHealthStatusQuery, GetIndicatorHealthStatusQueryVariables>;
|
||||
export const GetQueueMetricsDocument = gql`
|
||||
query GetQueueMetrics($queueName: String!, $timeRange: QueueMetricsTimeRange) {
|
||||
getQueueMetrics(queueName: $queueName, timeRange: $timeRange) {
|
||||
queueName
|
||||
timeRange
|
||||
workers
|
||||
details {
|
||||
failed
|
||||
completed
|
||||
waiting
|
||||
active
|
||||
delayed
|
||||
failureRate
|
||||
}
|
||||
data {
|
||||
id
|
||||
data {
|
||||
x
|
||||
y
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* __useGetQueueMetricsQuery__
|
||||
*
|
||||
* To run a query within a React component, call `useGetQueueMetricsQuery` and pass it any options that fit your needs.
|
||||
* When your component renders, `useGetQueueMetricsQuery` returns an object from Apollo Client that contains loading, error, and data properties
|
||||
* you can use to render your UI.
|
||||
*
|
||||
* @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options;
|
||||
*
|
||||
* @example
|
||||
* const { data, loading, error } = useGetQueueMetricsQuery({
|
||||
* variables: {
|
||||
* queueName: // value for 'queueName'
|
||||
* timeRange: // value for 'timeRange'
|
||||
* },
|
||||
* });
|
||||
*/
|
||||
export function useGetQueueMetricsQuery(baseOptions: Apollo.QueryHookOptions<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>) {
|
||||
const options = {...defaultOptions, ...baseOptions}
|
||||
return Apollo.useQuery<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>(GetQueueMetricsDocument, options);
|
||||
}
|
||||
export function useGetQueueMetricsLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>) {
|
||||
const options = {...defaultOptions, ...baseOptions}
|
||||
return Apollo.useLazyQuery<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>(GetQueueMetricsDocument, options);
|
||||
}
|
||||
export type GetQueueMetricsQueryHookResult = ReturnType<typeof useGetQueueMetricsQuery>;
|
||||
export type GetQueueMetricsLazyQueryHookResult = ReturnType<typeof useGetQueueMetricsLazyQuery>;
|
||||
export type GetQueueMetricsQueryResult = Apollo.QueryResult<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>;
|
||||
export const GetSystemHealthStatusDocument = gql`
|
||||
query GetSystemHealthStatus {
|
||||
getSystemHealthStatus {
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { SettingsAdminEnvVariablesTable } from '@/settings/admin-panel/components/SettingsAdminEnvVariablesTable';
|
||||
import { SettingsAdminTabSkeletonLoader } from '@/settings/admin-panel/components/SettingsAdminTabSkeletonLoader';
|
||||
import styled from '@emotion/styled';
|
||||
import { useState } from 'react';
|
||||
import { Button, H1Title, H1TitleFontColor, Section } from 'twenty-ui';
|
||||
@ -37,11 +38,10 @@ const StyledShowMoreButton = styled(Button)<{ isSelected?: boolean }>`
|
||||
`;
|
||||
|
||||
export const SettingsAdminEnvVariables = () => {
|
||||
const { data: environmentVariables } = useGetEnvironmentVariablesGroupedQuery(
|
||||
{
|
||||
const { data: environmentVariables, loading: environmentVariablesLoading } =
|
||||
useGetEnvironmentVariablesGroupedQuery({
|
||||
fetchPolicy: 'network-only',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
const [selectedGroup, setSelectedGroup] = useState<string | null>(null);
|
||||
|
||||
@ -64,6 +64,10 @@ export const SettingsAdminEnvVariables = () => {
|
||||
(group) => group.name === selectedGroup,
|
||||
);
|
||||
|
||||
if (environmentVariablesLoading) {
|
||||
return <SettingsAdminTabSkeletonLoader />;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Section>
|
||||
|
||||
@ -0,0 +1,17 @@
|
||||
import { useTheme } from '@emotion/react';
|
||||
import Skeleton, { SkeletonTheme } from 'react-loading-skeleton';
|
||||
|
||||
import { SKELETON_LOADER_HEIGHT_SIZES } from '@/activities/components/SkeletonLoader';
|
||||
|
||||
export const SettingsAdminTabSkeletonLoader = () => {
|
||||
const theme = useTheme();
|
||||
return (
|
||||
<SkeletonTheme
|
||||
baseColor={theme.background.tertiary}
|
||||
highlightColor={theme.background.transparent.lighter}
|
||||
borderRadius={4}
|
||||
>
|
||||
<Skeleton height={SKELETON_LOADER_HEIGHT_SIZES.standard.m} width={120} />
|
||||
</SkeletonTheme>
|
||||
);
|
||||
};
|
||||
@ -19,9 +19,7 @@ const StyledErrorMessage = styled.div`
|
||||
`;
|
||||
|
||||
export const DatabaseAndRedisHealthStatus = () => {
|
||||
const { indicatorHealth, loading } = useContext(
|
||||
SettingsAdminIndicatorHealthContext,
|
||||
);
|
||||
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
|
||||
|
||||
const formattedDetails = indicatorHealth.details
|
||||
? JSON.stringify(JSON.parse(indicatorHealth.details), null, 2)
|
||||
@ -33,7 +31,7 @@ export const DatabaseAndRedisHealthStatus = () => {
|
||||
|
||||
return (
|
||||
<Section>
|
||||
{isDatabaseOrRedisDown && !loading ? (
|
||||
{isDatabaseOrRedisDown ? (
|
||||
<StyledErrorMessage>
|
||||
{`${indicatorHealth.label} information is not available because the service is down`}
|
||||
</StyledErrorMessage>
|
||||
|
||||
@ -1,18 +1,27 @@
|
||||
import { SettingsAdminTabSkeletonLoader } from '@/settings/admin-panel/components/SettingsAdminTabSkeletonLoader';
|
||||
import { SettingsHealthStatusListCard } from '@/settings/admin-panel/health-status/components/SettingsHealthStatusListCard';
|
||||
import { H2Title, Section } from 'twenty-ui';
|
||||
import { useGetSystemHealthStatusQuery } from '~/generated/graphql';
|
||||
|
||||
export const SettingsAdminHealthStatus = () => {
|
||||
const { data, loading } = useGetSystemHealthStatusQuery({
|
||||
const { data, loading: loadingHealthStatus } = useGetSystemHealthStatusQuery({
|
||||
fetchPolicy: 'network-only',
|
||||
});
|
||||
|
||||
const services = data?.getSystemHealthStatus.services ?? [];
|
||||
|
||||
if (loadingHealthStatus) {
|
||||
return <SettingsAdminTabSkeletonLoader />;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Section>
|
||||
<H2Title title="Health Status" description="How your system is doing" />
|
||||
<SettingsHealthStatusListCard services={services} loading={loading} />
|
||||
<SettingsHealthStatusListCard
|
||||
services={services}
|
||||
loading={loadingHealthStatus}
|
||||
/>
|
||||
</Section>
|
||||
</>
|
||||
);
|
||||
|
||||
@ -1,105 +0,0 @@
|
||||
import { SettingsListCard } from '@/settings/components/SettingsListCard';
|
||||
import { Table } from '@/ui/layout/table/components/Table';
|
||||
import { TableCell } from '@/ui/layout/table/components/TableCell';
|
||||
import { TableRow } from '@/ui/layout/table/components/TableRow';
|
||||
import styled from '@emotion/styled';
|
||||
import { AnimatedExpandableContainer, Status } from 'twenty-ui';
|
||||
import {
|
||||
AdminPanelHealthServiceStatus,
|
||||
AdminPanelWorkerQueueHealth,
|
||||
} from '~/generated/graphql';
|
||||
|
||||
const StyledExpandedContent = styled.div`
|
||||
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
padding-top: ${({ theme }) => theme.spacing(1)};
|
||||
padding-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||
padding-right: ${({ theme }) => theme.spacing(3)};
|
||||
`;
|
||||
|
||||
const StyledContainer = styled.div`
|
||||
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
margin-top: ${({ theme }) => theme.spacing(5)};
|
||||
`;
|
||||
|
||||
const StyledTableRow = styled(TableRow)`
|
||||
height: ${({ theme }) => theme.spacing(6)};
|
||||
`;
|
||||
|
||||
const StyledQueueMetricsTitle = styled.div`
|
||||
color: ${({ theme }) => theme.font.color.primary};
|
||||
font-size: ${({ theme }) => theme.font.size.sm};
|
||||
font-weight: ${({ theme }) => theme.font.weight.medium};
|
||||
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||
`;
|
||||
|
||||
export const SettingsAdminQueueExpandableContainer = ({
|
||||
queues,
|
||||
selectedQueue,
|
||||
}: {
|
||||
queues: AdminPanelWorkerQueueHealth[];
|
||||
selectedQueue: string | null;
|
||||
}) => {
|
||||
const selectedQueueData = queues.find(
|
||||
(queue) => queue.queueName === selectedQueue,
|
||||
);
|
||||
|
||||
return (
|
||||
<AnimatedExpandableContainer
|
||||
isExpanded={!!selectedQueue}
|
||||
mode="fit-content"
|
||||
>
|
||||
{selectedQueueData && (
|
||||
<>
|
||||
<StyledContainer>
|
||||
<SettingsListCard
|
||||
items={[
|
||||
{ ...selectedQueueData, id: selectedQueueData.queueName },
|
||||
]}
|
||||
getItemLabel={(
|
||||
item: AdminPanelWorkerQueueHealth & { id: string },
|
||||
) => item.queueName}
|
||||
isLoading={false}
|
||||
RowRightComponent={({
|
||||
item,
|
||||
}: {
|
||||
item: AdminPanelWorkerQueueHealth;
|
||||
}) => (
|
||||
<Status
|
||||
color={
|
||||
item.status === AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
? 'green'
|
||||
: 'red'
|
||||
}
|
||||
text={item.status.toLowerCase()}
|
||||
weight="medium"
|
||||
/>
|
||||
)}
|
||||
/>
|
||||
</StyledContainer>
|
||||
<StyledQueueMetricsTitle> Metrics:</StyledQueueMetricsTitle>
|
||||
<StyledExpandedContent>
|
||||
<Table>
|
||||
<StyledTableRow>
|
||||
<TableCell align="left">Workers</TableCell>
|
||||
<TableCell align="right">{selectedQueueData.workers}</TableCell>
|
||||
</StyledTableRow>
|
||||
{Object.entries(selectedQueueData.metrics)
|
||||
.filter(([key]) => key !== '__typename')
|
||||
.map(([key, value]) => (
|
||||
<StyledTableRow key={key}>
|
||||
<TableCell align="left">
|
||||
{key.charAt(0).toUpperCase() + key.slice(1)}
|
||||
</TableCell>
|
||||
<TableCell align="right">{value}</TableCell>
|
||||
</StyledTableRow>
|
||||
))}
|
||||
</Table>
|
||||
</StyledExpandedContent>
|
||||
</>
|
||||
)}
|
||||
</AnimatedExpandableContainer>
|
||||
);
|
||||
};
|
||||
@ -1,52 +0,0 @@
|
||||
import styled from '@emotion/styled';
|
||||
import { Button } from 'twenty-ui';
|
||||
import {
|
||||
AdminPanelHealthServiceStatus,
|
||||
AdminPanelWorkerQueueHealth,
|
||||
} from '~/generated/graphql';
|
||||
|
||||
const StyledQueueButtonsRow = styled.div`
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: ${({ theme }) => theme.spacing(2)};
|
||||
margin-top: ${({ theme }) => theme.spacing(6)};
|
||||
`;
|
||||
|
||||
const StyledQueueHealthButton = styled(Button)<{
|
||||
isSelected?: boolean;
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
}>`
|
||||
${({ isSelected, theme, status }) =>
|
||||
isSelected &&
|
||||
`
|
||||
background-color: ${
|
||||
status === AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
? theme.tag.background.green
|
||||
: theme.tag.background.red
|
||||
};
|
||||
`}
|
||||
`;
|
||||
export const SettingsAdminQueueHealthButtons = ({
|
||||
queues,
|
||||
selectedQueue,
|
||||
toggleQueueVisibility,
|
||||
}: {
|
||||
queues: AdminPanelWorkerQueueHealth[];
|
||||
selectedQueue: string | null;
|
||||
toggleQueueVisibility: (queueName: string) => void;
|
||||
}) => {
|
||||
return (
|
||||
<StyledQueueButtonsRow>
|
||||
{queues.map((queue) => (
|
||||
<StyledQueueHealthButton
|
||||
key={queue.queueName}
|
||||
onClick={() => toggleQueueVisibility(queue.queueName)}
|
||||
title={queue.queueName}
|
||||
variant="secondary"
|
||||
isSelected={selectedQueue === queue.queueName}
|
||||
status={queue.status}
|
||||
/>
|
||||
))}
|
||||
</StyledQueueButtonsRow>
|
||||
);
|
||||
};
|
||||
@ -1,16 +1,8 @@
|
||||
import { SettingsAdminQueueExpandableContainer } from '@/settings/admin-panel/health-status/components/SettingsAdminQueueExpandableContainer';
|
||||
import { SettingsAdminQueueHealthButtons } from '@/settings/admin-panel/health-status/components/SettingsAdminQueueHealthButtons';
|
||||
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
|
||||
import { WorkerQueueMetricsSection } from '@/settings/admin-panel/health-status/components/WorkerQueueMetricsSection';
|
||||
import styled from '@emotion/styled';
|
||||
import { useContext, useState } from 'react';
|
||||
import { H2Title, Section } from 'twenty-ui';
|
||||
import { useContext } from 'react';
|
||||
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
|
||||
|
||||
const StyledTitleContainer = styled.div`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
flex: 1;
|
||||
`;
|
||||
import { SettingsAdminIndicatorHealthContext } from '../contexts/SettingsAdminIndicatorHealthContext';
|
||||
|
||||
const StyledErrorMessage = styled.div`
|
||||
color: ${({ theme }) => theme.color.red};
|
||||
@ -18,45 +10,23 @@ const StyledErrorMessage = styled.div`
|
||||
`;
|
||||
|
||||
export const WorkerHealthStatus = () => {
|
||||
const { indicatorHealth, loading } = useContext(
|
||||
SettingsAdminIndicatorHealthContext,
|
||||
);
|
||||
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
|
||||
|
||||
const isWorkerDown =
|
||||
!indicatorHealth.status ||
|
||||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
|
||||
|
||||
const [selectedQueue, setSelectedQueue] = useState<string | null>(null);
|
||||
|
||||
const toggleQueueVisibility = (queueName: string) => {
|
||||
setSelectedQueue(selectedQueue === queueName ? null : queueName);
|
||||
};
|
||||
|
||||
return (
|
||||
<Section>
|
||||
<StyledTitleContainer>
|
||||
<H2Title
|
||||
title="Queue Status"
|
||||
description="Background job processing status and metrics"
|
||||
/>
|
||||
</StyledTitleContainer>
|
||||
{isWorkerDown && !loading ? (
|
||||
<>
|
||||
{isWorkerDown ? (
|
||||
<StyledErrorMessage>
|
||||
Queue information is not available because the worker is down
|
||||
</StyledErrorMessage>
|
||||
) : (
|
||||
<>
|
||||
<SettingsAdminQueueHealthButtons
|
||||
queues={indicatorHealth.queues ?? []}
|
||||
selectedQueue={selectedQueue}
|
||||
toggleQueueVisibility={toggleQueueVisibility}
|
||||
/>
|
||||
<SettingsAdminQueueExpandableContainer
|
||||
queues={indicatorHealth.queues ?? []}
|
||||
selectedQueue={selectedQueue}
|
||||
/>
|
||||
</>
|
||||
(indicatorHealth.queues ?? []).map((queue) => (
|
||||
<WorkerQueueMetricsSection key={queue.queueName} queue={queue} />
|
||||
))
|
||||
)}
|
||||
</Section>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
@ -0,0 +1,315 @@
|
||||
import { useTheme } from '@emotion/react';
|
||||
import styled from '@emotion/styled';
|
||||
import { ResponsiveLine } from '@nivo/line';
|
||||
|
||||
import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar';
|
||||
import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
|
||||
import { Select } from '@/ui/input/components/Select';
|
||||
import { Table } from '@/ui/layout/table/components/Table';
|
||||
import { TableCell } from '@/ui/layout/table/components/TableCell';
|
||||
import { TableRow } from '@/ui/layout/table/components/TableRow';
|
||||
import {
|
||||
QueueMetricsTimeRange,
|
||||
useGetQueueMetricsQuery,
|
||||
} from '~/generated/graphql';
|
||||
|
||||
const StyledTableRow = styled(TableRow)`
|
||||
height: ${({ theme }) => theme.spacing(6)};
|
||||
`;
|
||||
|
||||
const StyledQueueMetricsTitle = styled.div`
|
||||
color: ${({ theme }) => theme.font.color.primary};
|
||||
font-size: ${({ theme }) => theme.font.size.sm};
|
||||
font-weight: ${({ theme }) => theme.font.weight.medium};
|
||||
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||
`;
|
||||
|
||||
const StyledGraphContainer = styled.div`
|
||||
background-color: ${({ theme }) => theme.background.tertiary};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
height: 230px;
|
||||
margin-bottom: ${({ theme }) => theme.spacing(4)};
|
||||
padding-top: ${({ theme }) => theme.spacing(4)};
|
||||
padding-bottom: ${({ theme }) => theme.spacing(2)};
|
||||
width: 100%;
|
||||
`;
|
||||
|
||||
const StyledQueueMetricsContainer = styled.div`
|
||||
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
padding-top: ${({ theme }) => theme.spacing(1)};
|
||||
padding-bottom: ${({ theme }) => theme.spacing(3)};
|
||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||
padding-right: ${({ theme }) => theme.spacing(3)};
|
||||
`;
|
||||
|
||||
const StyledGraphControls = styled.div`
|
||||
display: flex;
|
||||
gap: ${({ theme }) => theme.spacing(2)};
|
||||
justify-content: flex-end;
|
||||
margin-bottom: ${({ theme }) => theme.spacing(2)};
|
||||
margin-top: ${({ theme }) => theme.spacing(1)};
|
||||
`;
|
||||
|
||||
const StyledNoDataMessage = styled.div`
|
||||
align-items: center;
|
||||
color: ${({ theme }) => theme.font.color.light};
|
||||
display: flex;
|
||||
height: 100%;
|
||||
justify-content: center;
|
||||
`;
|
||||
|
||||
const StyledTooltipContainer = styled.div`
|
||||
background-color: ${({ theme }) => theme.background.secondary};
|
||||
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
||||
padding: ${({ theme }) => theme.spacing(2)};
|
||||
font-size: ${({ theme }) => theme.font.size.sm};
|
||||
`;
|
||||
|
||||
const StyledTooltipItem = styled.div<{ color: string }>`
|
||||
align-items: center;
|
||||
color: ${({ theme }) => theme.font.color.primary};
|
||||
display: flex;
|
||||
padding: ${({ theme }) => theme.spacing(0.5)} 0;
|
||||
`;
|
||||
|
||||
const StyledTooltipColorSquare = styled.div<{ color: string }>`
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
background-color: ${({ color }) => color};
|
||||
margin-right: ${({ theme }) => theme.spacing(1)};
|
||||
`;
|
||||
|
||||
const StyledTooltipValue = styled.span`
|
||||
font-weight: ${({ theme }) => theme.font.weight.medium};
|
||||
`;
|
||||
|
||||
type WorkerMetricsGraphProps = {
|
||||
queueName: string;
|
||||
timeRange: QueueMetricsTimeRange;
|
||||
onTimeRangeChange: (range: QueueMetricsTimeRange) => void;
|
||||
};
|
||||
|
||||
export const WorkerMetricsGraph = ({
|
||||
queueName,
|
||||
timeRange,
|
||||
onTimeRangeChange,
|
||||
}: WorkerMetricsGraphProps) => {
|
||||
const theme = useTheme();
|
||||
const { enqueueSnackBar } = useSnackBar();
|
||||
|
||||
const { loading, data } = useGetQueueMetricsQuery({
|
||||
variables: {
|
||||
queueName,
|
||||
timeRange,
|
||||
},
|
||||
fetchPolicy: 'no-cache',
|
||||
onError: (error) => {
|
||||
enqueueSnackBar(`Error fetching worker metrics: ${error.message}`, {
|
||||
variant: SnackBarVariant.Error,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const metricsData = data?.getQueueMetrics?.data || [];
|
||||
const hasData =
|
||||
metricsData.length > 0 &&
|
||||
metricsData.some((series) => series.data.length > 0);
|
||||
|
||||
const metricsDetails = {
|
||||
workers: data?.getQueueMetrics?.workers,
|
||||
...data?.getQueueMetrics?.details,
|
||||
};
|
||||
|
||||
const getMaxYValue = () => {
|
||||
if (!hasData) return 2;
|
||||
|
||||
let maxValue = 0;
|
||||
metricsData.forEach((series) => {
|
||||
series.data.forEach((point) => {
|
||||
if (typeof point.y === 'number' && point.y > maxValue) {
|
||||
maxValue = point.y;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return maxValue === 0 ? 2 : maxValue * 1.1;
|
||||
};
|
||||
|
||||
const getAxisLabel = () => {
|
||||
switch (timeRange) {
|
||||
case QueueMetricsTimeRange.OneHour:
|
||||
return 'Last 1 Hour (oldest → newest)';
|
||||
case QueueMetricsTimeRange.FourHours:
|
||||
return 'Last 4 Hours (oldest → newest)';
|
||||
case QueueMetricsTimeRange.TwelveHours:
|
||||
return 'Last 12 Hours (oldest → newest)';
|
||||
case QueueMetricsTimeRange.OneDay:
|
||||
return 'Last 24 Hours (oldest → newest)';
|
||||
case QueueMetricsTimeRange.SevenDays:
|
||||
return 'Last 7 Days (oldest → newest)';
|
||||
default:
|
||||
return 'Recent Events (oldest → newest)';
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<StyledGraphControls>
|
||||
<Select
|
||||
dropdownId={`timerange-${queueName}`}
|
||||
value={timeRange}
|
||||
options={[
|
||||
{ value: QueueMetricsTimeRange.SevenDays, label: 'This week' },
|
||||
{ value: QueueMetricsTimeRange.OneDay, label: 'Today' },
|
||||
{
|
||||
value: QueueMetricsTimeRange.TwelveHours,
|
||||
label: 'Last 12 hours',
|
||||
},
|
||||
{ value: QueueMetricsTimeRange.FourHours, label: 'Last 4 hours' },
|
||||
{ value: QueueMetricsTimeRange.OneHour, label: 'Last 1 hour' },
|
||||
]}
|
||||
onChange={onTimeRangeChange}
|
||||
needIconCheck
|
||||
/>
|
||||
</StyledGraphControls>
|
||||
|
||||
<StyledGraphContainer>
|
||||
{loading ? (
|
||||
<StyledNoDataMessage>Loading metrics data...</StyledNoDataMessage>
|
||||
) : hasData ? (
|
||||
<ResponsiveLine
|
||||
data={metricsData}
|
||||
curve="monotoneX"
|
||||
enableArea={true}
|
||||
colors={[theme.color.green, theme.color.red]}
|
||||
theme={{
|
||||
text: {
|
||||
fill: theme.font.color.light,
|
||||
fontSize: theme.font.size.sm,
|
||||
fontFamily: theme.font.family,
|
||||
},
|
||||
axis: {
|
||||
domain: {
|
||||
line: {
|
||||
stroke: theme.border.color.strong,
|
||||
},
|
||||
},
|
||||
ticks: {
|
||||
line: {
|
||||
stroke: theme.border.color.strong,
|
||||
},
|
||||
},
|
||||
},
|
||||
grid: {
|
||||
line: {
|
||||
stroke: theme.border.color.medium,
|
||||
},
|
||||
},
|
||||
crosshair: {
|
||||
line: {
|
||||
stroke: theme.font.color.primary,
|
||||
strokeDasharray: '2 2',
|
||||
},
|
||||
},
|
||||
}}
|
||||
margin={{ top: 40, right: 30, bottom: 40, left: 50 }}
|
||||
xScale={{
|
||||
type: 'linear',
|
||||
min: 0,
|
||||
max: 'auto',
|
||||
}}
|
||||
yScale={{
|
||||
type: 'linear',
|
||||
min: 0,
|
||||
max: getMaxYValue(),
|
||||
stacked: false,
|
||||
}}
|
||||
axisBottom={{
|
||||
legend: getAxisLabel(),
|
||||
legendOffset: 30,
|
||||
legendPosition: 'middle',
|
||||
tickSize: 5,
|
||||
tickPadding: 5,
|
||||
tickValues: 5,
|
||||
format: () => '',
|
||||
}}
|
||||
axisLeft={{
|
||||
tickSize: 6,
|
||||
tickPadding: 5,
|
||||
tickValues: 4,
|
||||
legend: 'Count',
|
||||
legendOffset: -40,
|
||||
legendPosition: 'middle',
|
||||
}}
|
||||
enableGridX={false}
|
||||
gridYValues={4}
|
||||
pointSize={0}
|
||||
enableSlices="x"
|
||||
sliceTooltip={({ slice }) => (
|
||||
<StyledTooltipContainer>
|
||||
{slice.points.map((point) => (
|
||||
<StyledTooltipItem key={point.id} color={point.serieColor}>
|
||||
<StyledTooltipColorSquare color={point.serieColor} />
|
||||
<span>
|
||||
{point.serieId}:{' '}
|
||||
<StyledTooltipValue>
|
||||
{String(point.data.y)}
|
||||
</StyledTooltipValue>
|
||||
</span>
|
||||
</StyledTooltipItem>
|
||||
))}
|
||||
</StyledTooltipContainer>
|
||||
)}
|
||||
useMesh={true}
|
||||
legends={[
|
||||
{
|
||||
anchor: 'top-right',
|
||||
direction: 'row',
|
||||
justify: false,
|
||||
translateX: 0,
|
||||
translateY: -40,
|
||||
itemsSpacing: 10,
|
||||
itemDirection: 'left-to-right',
|
||||
itemWidth: 100,
|
||||
itemHeight: 20,
|
||||
symbolSize: 12,
|
||||
symbolShape: 'square',
|
||||
},
|
||||
]}
|
||||
/>
|
||||
) : (
|
||||
<StyledNoDataMessage>No metrics data available</StyledNoDataMessage>
|
||||
)}
|
||||
</StyledGraphContainer>
|
||||
{metricsDetails && (
|
||||
<>
|
||||
<StyledQueueMetricsTitle>Metrics:</StyledQueueMetricsTitle>
|
||||
<StyledQueueMetricsContainer>
|
||||
<Table>
|
||||
{Object.entries(metricsDetails)
|
||||
.filter(([key]) => key !== '__typename')
|
||||
.map(([key, value]) => (
|
||||
<StyledTableRow key={key}>
|
||||
<TableCell align="left">
|
||||
{key.charAt(0).toUpperCase() + key.slice(1)}
|
||||
</TableCell>
|
||||
<TableCell align="right">
|
||||
{typeof value === 'number'
|
||||
? value
|
||||
: Array.isArray(value)
|
||||
? value.length
|
||||
: String(value)}
|
||||
</TableCell>
|
||||
</StyledTableRow>
|
||||
))}
|
||||
</Table>
|
||||
</StyledQueueMetricsContainer>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,28 @@
|
||||
import { useState } from 'react';
|
||||
import { H2Title, Section } from 'twenty-ui';
|
||||
import {
|
||||
AdminPanelWorkerQueueHealth,
|
||||
QueueMetricsTimeRange,
|
||||
} from '~/generated/graphql';
|
||||
import { WorkerMetricsGraph } from './WorkerMetricsGraph';
|
||||
|
||||
type WorkerQueueMetricsSectionProps = {
|
||||
queue: AdminPanelWorkerQueueHealth;
|
||||
};
|
||||
|
||||
export const WorkerQueueMetricsSection = ({
|
||||
queue,
|
||||
}: WorkerQueueMetricsSectionProps) => {
|
||||
const [timeRange, setTimeRange] = useState(QueueMetricsTimeRange.OneHour);
|
||||
|
||||
return (
|
||||
<Section>
|
||||
<H2Title title={queue.queueName} description="Queue performance" />
|
||||
<WorkerMetricsGraph
|
||||
queueName={queue.queueName}
|
||||
timeRange={timeRange}
|
||||
onTimeRangeChange={setTimeRange}
|
||||
/>
|
||||
</Section>
|
||||
);
|
||||
};
|
||||
@ -6,7 +6,6 @@ import {
|
||||
|
||||
type SettingsAdminIndicatorHealthContextType = {
|
||||
indicatorHealth: AdminPanelHealthServiceData;
|
||||
loading: boolean;
|
||||
};
|
||||
|
||||
export const SettingsAdminIndicatorHealthContext =
|
||||
@ -19,5 +18,4 @@ export const SettingsAdminIndicatorHealthContext =
|
||||
details: '',
|
||||
queues: [],
|
||||
},
|
||||
loading: false,
|
||||
});
|
||||
|
||||
@ -12,15 +12,6 @@ export const GET_INDICATOR_HEALTH_STATUS = gql`
|
||||
id
|
||||
queueName
|
||||
status
|
||||
workers
|
||||
metrics {
|
||||
failed
|
||||
completed
|
||||
waiting
|
||||
active
|
||||
delayed
|
||||
prioritized
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,29 @@
|
||||
import { gql } from '@apollo/client';
|
||||
|
||||
export const GET_QUEUE_METRICS = gql`
|
||||
query GetQueueMetrics(
|
||||
$queueName: String!
|
||||
$timeRange: QueueMetricsTimeRange
|
||||
) {
|
||||
getQueueMetrics(queueName: $queueName, timeRange: $timeRange) {
|
||||
queueName
|
||||
timeRange
|
||||
workers
|
||||
details {
|
||||
failed
|
||||
completed
|
||||
waiting
|
||||
active
|
||||
delayed
|
||||
failureRate
|
||||
}
|
||||
data {
|
||||
id
|
||||
data {
|
||||
x
|
||||
y
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
@ -2,6 +2,7 @@ import { SettingsAdminHealthStatusRightContainer } from '@/settings/admin-panel/
|
||||
import { SettingsAdminIndicatorHealthStatusContent } from '@/settings/admin-panel/health-status/components/SettingsAdminIndicatorHealthStatusContent';
|
||||
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
|
||||
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
|
||||
import { SettingsSkeletonLoader } from '@/settings/components/SettingsSkeletonLoader';
|
||||
import { SettingsPath } from '@/types/SettingsPath';
|
||||
import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer';
|
||||
import styled from '@emotion/styled';
|
||||
@ -22,12 +23,17 @@ const StyledH2Title = styled(H2Title)`
|
||||
export const SettingsAdminIndicatorHealthStatus = () => {
|
||||
const { t } = useLingui();
|
||||
const { indicatorId } = useParams();
|
||||
const { data, loading } = useGetIndicatorHealthStatusQuery({
|
||||
variables: {
|
||||
indicatorId: indicatorId as HealthIndicatorId,
|
||||
},
|
||||
fetchPolicy: 'network-only',
|
||||
});
|
||||
const { data, loading: loadingIndicatorHealthStatus } =
|
||||
useGetIndicatorHealthStatusQuery({
|
||||
variables: {
|
||||
indicatorId: indicatorId as HealthIndicatorId,
|
||||
},
|
||||
fetchPolicy: 'network-only',
|
||||
});
|
||||
|
||||
if (loadingIndicatorHealthStatus) {
|
||||
return <SettingsSkeletonLoader />;
|
||||
}
|
||||
|
||||
return (
|
||||
<SubMenuTopBarContainer
|
||||
@ -60,7 +66,6 @@ export const SettingsAdminIndicatorHealthStatus = () => {
|
||||
details: data?.getIndicatorHealthStatus?.details,
|
||||
queues: data?.getIndicatorHealthStatus?.queues,
|
||||
},
|
||||
loading: loading,
|
||||
}}
|
||||
>
|
||||
<Section>
|
||||
|
||||
@ -1,15 +1,24 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { Queue } from 'bullmq';
|
||||
import { Redis } from 'ioredis';
|
||||
|
||||
import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service';
|
||||
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
||||
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
jest.mock('bullmq');
|
||||
|
||||
describe('AdminPanelHealthService', () => {
|
||||
let service: AdminPanelHealthService;
|
||||
@ -17,12 +26,25 @@ describe('AdminPanelHealthService', () => {
|
||||
let redisHealth: jest.Mocked<RedisHealthIndicator>;
|
||||
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
|
||||
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
|
||||
let redisClient: jest.Mocked<RedisClientService>;
|
||||
let environmentService: jest.Mocked<EnvironmentService>;
|
||||
let loggerSpy: jest.SpyInstance;
|
||||
|
||||
beforeEach(async () => {
|
||||
databaseHealth = { isHealthy: jest.fn() } as any;
|
||||
redisHealth = { isHealthy: jest.fn() } as any;
|
||||
workerHealth = { isHealthy: jest.fn() } as any;
|
||||
workerHealth = { isHealthy: jest.fn(), getQueueDetails: jest.fn() } as any;
|
||||
connectedAccountHealth = { isHealthy: jest.fn() } as any;
|
||||
redisClient = {
|
||||
getClient: jest.fn().mockReturnValue({} as Redis),
|
||||
} as any;
|
||||
environmentService = { get: jest.fn() } as any;
|
||||
|
||||
(Queue as unknown as jest.Mock) = jest.fn().mockImplementation(() => ({
|
||||
getMetrics: jest.fn(),
|
||||
getWorkers: jest.fn(),
|
||||
close: jest.fn(),
|
||||
}));
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
@ -31,10 +53,21 @@ describe('AdminPanelHealthService', () => {
|
||||
{ provide: RedisHealthIndicator, useValue: redisHealth },
|
||||
{ provide: WorkerHealthIndicator, useValue: workerHealth },
|
||||
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
|
||||
{ provide: RedisClientService, useValue: redisClient },
|
||||
{ provide: EnvironmentService, useValue: environmentService },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<AdminPanelHealthService>(AdminPanelHealthService);
|
||||
|
||||
loggerSpy = jest
|
||||
.spyOn(service['logger'], 'error')
|
||||
.mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
loggerSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
@ -62,8 +95,9 @@ describe('AdminPanelHealthService', () => {
|
||||
delayed: 4,
|
||||
failed: 3,
|
||||
waiting: 0,
|
||||
prioritized: 0,
|
||||
failureRate: 0.3,
|
||||
},
|
||||
status: 'up',
|
||||
},
|
||||
],
|
||||
},
|
||||
@ -209,26 +243,12 @@ describe('AdminPanelHealthService', () => {
|
||||
{
|
||||
queueName: 'queue1',
|
||||
workers: 2,
|
||||
metrics: {
|
||||
active: 1,
|
||||
completed: 10,
|
||||
delayed: 0,
|
||||
failed: 2,
|
||||
waiting: 5,
|
||||
prioritized: 1,
|
||||
},
|
||||
status: 'up',
|
||||
},
|
||||
{
|
||||
queueName: 'queue2',
|
||||
workers: 0,
|
||||
metrics: {
|
||||
active: 0,
|
||||
completed: 5,
|
||||
delayed: 0,
|
||||
failed: 1,
|
||||
waiting: 2,
|
||||
prioritized: 0,
|
||||
},
|
||||
status: 'up',
|
||||
},
|
||||
];
|
||||
|
||||
@ -248,8 +268,8 @@ describe('AdminPanelHealthService', () => {
|
||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||
details: undefined,
|
||||
queues: mockQueues.map((queue) => ({
|
||||
...queue,
|
||||
id: `worker-${queue.queueName}`,
|
||||
queueName: queue.queueName,
|
||||
status:
|
||||
queue.workers > 0
|
||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
@ -281,4 +301,305 @@ describe('AdminPanelHealthService', () => {
|
||||
).rejects.toThrow('Health indicator not found: invalid');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getQueueMetrics', () => {
|
||||
const mockQueue = {
|
||||
getMetrics: jest.fn(),
|
||||
getWorkers: jest.fn(),
|
||||
close: jest.fn(),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
redisClient.getClient.mockReturnValue({} as Redis);
|
||||
(Queue as unknown as jest.Mock).mockImplementation(() => mockQueue);
|
||||
});
|
||||
|
||||
it('should return metrics data for a queue with correct data transformation', async () => {
|
||||
const mockCompletedData = Array(240)
|
||||
.fill(0)
|
||||
.map((_, i) => i);
|
||||
const mockFailedData = Array(240)
|
||||
.fill(0)
|
||||
.map((_, i) => i * 0.1);
|
||||
|
||||
workerHealth.getQueueDetails.mockResolvedValue({
|
||||
queueName: 'test-queue',
|
||||
workers: 1,
|
||||
status: 'up',
|
||||
metrics: {
|
||||
active: 1,
|
||||
completed: 30,
|
||||
failed: 3,
|
||||
waiting: 0,
|
||||
delayed: 0,
|
||||
failureRate: 9.1,
|
||||
completedData: mockCompletedData,
|
||||
failedData: mockFailedData,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getQueueMetrics(
|
||||
MessageQueue.messagingQueue,
|
||||
QueueMetricsTimeRange.FourHours,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
queueName: MessageQueue.messagingQueue,
|
||||
timeRange: QueueMetricsTimeRange.FourHours,
|
||||
workers: 1,
|
||||
details: expect.any(Object),
|
||||
data: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: 'Completed Jobs',
|
||||
data: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
x: expect.any(Number),
|
||||
y: expect.any(Number),
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: 'Failed Jobs',
|
||||
data: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
x: expect.any(Number),
|
||||
y: expect.any(Number),
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle empty metrics data', async () => {
|
||||
workerHealth.getQueueDetails.mockResolvedValue(null);
|
||||
|
||||
const result = await service.getQueueMetrics(
|
||||
MessageQueue.messagingQueue,
|
||||
QueueMetricsTimeRange.FourHours,
|
||||
);
|
||||
|
||||
expect(result.data).toHaveLength(2);
|
||||
expect(result.data[0].data).toHaveLength(240);
|
||||
expect(result.data[1].data).toHaveLength(240);
|
||||
});
|
||||
|
||||
it('should handle metrics service errors', async () => {
|
||||
workerHealth.getQueueDetails.mockRejectedValue(
|
||||
new Error('Metrics error'),
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.getQueueMetrics(
|
||||
MessageQueue.messagingQueue,
|
||||
QueueMetricsTimeRange.FourHours,
|
||||
),
|
||||
).rejects.toThrow('Metrics error');
|
||||
|
||||
expect(loggerSpy).toHaveBeenCalledWith(
|
||||
'Error getting metrics for messaging-queue: Metrics error',
|
||||
);
|
||||
});
|
||||
|
||||
describe('backfilling behavior', () => {
|
||||
it('should handle partial data with correct historical backfilling', async () => {
|
||||
// Test with 40 recent points for 4-hour range (needs 240 points)
|
||||
const partialData = Array(40)
|
||||
.fill(0)
|
||||
.map((_, i) => i + 1);
|
||||
|
||||
workerHealth.getQueueDetails.mockResolvedValue({
|
||||
queueName: 'test-queue',
|
||||
workers: 1,
|
||||
status: 'up',
|
||||
metrics: {
|
||||
failed: 0,
|
||||
completed: 0,
|
||||
waiting: 0,
|
||||
active: 0,
|
||||
delayed: 0,
|
||||
failureRate: 0,
|
||||
completedData: partialData,
|
||||
failedData: partialData,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getQueueMetrics(
|
||||
MessageQueue.messagingQueue,
|
||||
QueueMetricsTimeRange.FourHours,
|
||||
);
|
||||
|
||||
// Should have 240 total points
|
||||
expect(result.data[0].data).toHaveLength(240);
|
||||
|
||||
// First 200 points should be zero (historical backfill)
|
||||
const historicalPoints = result.data[0].data.slice(0, 200);
|
||||
|
||||
expect(historicalPoints.every((point) => point.y === 0)).toBe(true);
|
||||
|
||||
// Last 40 points should be actual data
|
||||
const actualDataPoints = result.data[0].data.slice(200);
|
||||
|
||||
expect(actualDataPoints.every((point) => point.y > 0)).toBe(true);
|
||||
|
||||
// Verify chronological order (increasing values)
|
||||
const nonZeroValues = actualDataPoints.map((point) => point.y);
|
||||
|
||||
for (let i = 1; i < nonZeroValues.length; i++) {
|
||||
expect(nonZeroValues[i]).toBeGreaterThan(nonZeroValues[i - 1]);
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle completely empty data with full backfilling', async () => {
|
||||
workerHealth.getQueueDetails.mockResolvedValue({
|
||||
queueName: 'test-queue',
|
||||
workers: 1,
|
||||
status: 'up',
|
||||
metrics: {
|
||||
failed: 0,
|
||||
completed: 0,
|
||||
waiting: 0,
|
||||
active: 0,
|
||||
delayed: 0,
|
||||
failureRate: 0,
|
||||
completedData: [],
|
||||
failedData: [],
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getQueueMetrics(
|
||||
MessageQueue.messagingQueue,
|
||||
QueueMetricsTimeRange.OneHour,
|
||||
);
|
||||
|
||||
// Should have 60 points for one hour
|
||||
expect(result.data[0].data).toHaveLength(60);
|
||||
// All points should be zero
|
||||
expect(result.data[0].data.every((point) => point.y === 0)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sampling behavior', () => {
|
||||
it('should correctly sample data for different time ranges', async () => {
|
||||
const testCases = [
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.OneHour,
|
||||
expectedPoints: 60,
|
||||
samplingFactor: 1,
|
||||
},
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.FourHours,
|
||||
expectedPoints: 240,
|
||||
samplingFactor: 1,
|
||||
},
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.OneDay,
|
||||
expectedPoints: 240,
|
||||
samplingFactor: 6,
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of testCases) {
|
||||
// Create test data with non-zero values
|
||||
const testData = Array(testCase.expectedPoints * 2)
|
||||
.fill(0)
|
||||
.map((_, i) => i + 1); // Start from 1 to avoid zero values
|
||||
|
||||
workerHealth.getQueueDetails.mockResolvedValue({
|
||||
queueName: 'test-queue',
|
||||
workers: 1,
|
||||
status: 'up',
|
||||
metrics: {
|
||||
failed: 0,
|
||||
completed: 0,
|
||||
waiting: 0,
|
||||
active: 0,
|
||||
delayed: 0,
|
||||
failureRate: 0,
|
||||
completedData: testData,
|
||||
failedData: testData,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.getQueueMetrics(
|
||||
MessageQueue.messagingQueue,
|
||||
testCase.timeRange,
|
||||
);
|
||||
|
||||
expect(result.data[0].data).toHaveLength(testCase.expectedPoints);
|
||||
|
||||
if (testCase.samplingFactor > 1) {
|
||||
const sampledData = result.data[0].data;
|
||||
|
||||
for (let i = 0; i < sampledData.length; i++) {
|
||||
const start = i * testCase.samplingFactor;
|
||||
const end = start + testCase.samplingFactor;
|
||||
const originalDataSlice = testData.slice(start, end);
|
||||
|
||||
if (originalDataSlice.length > 0) {
|
||||
// Add this check
|
||||
const maxInSlice = Math.max(...originalDataSlice);
|
||||
|
||||
expect(sampledData[i].y).toBeLessThanOrEqual(maxInSlice);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getPointsConfiguration', () => {
|
||||
const testCases = [
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.OneHour,
|
||||
expected: {
|
||||
pointsNeeded: 60,
|
||||
samplingFactor: 1,
|
||||
targetVisualizationPoints: 240,
|
||||
},
|
||||
},
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.FourHours,
|
||||
expected: {
|
||||
pointsNeeded: 240,
|
||||
samplingFactor: 1,
|
||||
targetVisualizationPoints: 240,
|
||||
},
|
||||
},
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.TwelveHours,
|
||||
expected: {
|
||||
pointsNeeded: 720,
|
||||
samplingFactor: 3,
|
||||
targetVisualizationPoints: 240,
|
||||
},
|
||||
},
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.OneDay,
|
||||
expected: {
|
||||
pointsNeeded: 1440,
|
||||
samplingFactor: 6,
|
||||
targetVisualizationPoints: 240,
|
||||
},
|
||||
},
|
||||
{
|
||||
timeRange: QueueMetricsTimeRange.SevenDays,
|
||||
expected: {
|
||||
pointsNeeded: 10080,
|
||||
samplingFactor: 42,
|
||||
targetVisualizationPoints: 240,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
testCases.forEach(({ timeRange, expected }) => {
|
||||
it(`should return correct parameters for ${timeRange}`, () => {
|
||||
const result = service['getPointsConfiguration'](timeRange as any);
|
||||
|
||||
expect(result).toEqual(expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,23 +1,33 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { HealthIndicatorResult, HealthIndicatorStatus } from '@nestjs/terminus';
|
||||
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
||||
import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto';
|
||||
import { QueueMetricsData } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-data.dto';
|
||||
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
@Injectable()
|
||||
export class AdminPanelHealthService {
|
||||
private readonly logger = new Logger(AdminPanelHealthService.name);
|
||||
|
||||
constructor(
|
||||
private readonly databaseHealth: DatabaseHealthIndicator,
|
||||
private readonly redisHealth: RedisHealthIndicator,
|
||||
private readonly workerHealth: WorkerHealthIndicator,
|
||||
private readonly connectedAccountHealth: ConnectedAccountHealth,
|
||||
private readonly redisClient: RedisClientService,
|
||||
) {}
|
||||
|
||||
private readonly healthIndicators = {
|
||||
@ -93,8 +103,8 @@ export class AdminPanelHealthService {
|
||||
return {
|
||||
...indicatorStatus,
|
||||
queues: (indicatorStatus?.queues ?? []).map((queue) => ({
|
||||
...queue,
|
||||
id: `${indicatorId}-${queue.queueName}`,
|
||||
queueName: queue.queueName,
|
||||
status:
|
||||
queue.workers > 0
|
||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||
@ -144,4 +154,166 @@ export class AdminPanelHealthService {
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async getQueueMetrics(
|
||||
queueName: MessageQueue,
|
||||
timeRange: QueueMetricsTimeRange = QueueMetricsTimeRange.OneDay,
|
||||
): Promise<QueueMetricsData> {
|
||||
const redis = this.redisClient.getClient();
|
||||
const queue = new Queue(queueName, { connection: redis });
|
||||
|
||||
try {
|
||||
const { pointsNeeded, samplingFactor } =
|
||||
this.getPointsConfiguration(timeRange);
|
||||
|
||||
const queueDetails = await this.workerHealth.getQueueDetails(queueName, {
|
||||
pointsNeeded,
|
||||
});
|
||||
|
||||
const completedMetricsArray = queueDetails?.metrics?.completedData;
|
||||
const failedMetricsArray = queueDetails?.metrics?.failedData;
|
||||
|
||||
const completedMetrics = this.extractMetricsData(
|
||||
completedMetricsArray,
|
||||
pointsNeeded,
|
||||
samplingFactor,
|
||||
);
|
||||
|
||||
const failedMetrics = this.extractMetricsData(
|
||||
failedMetricsArray,
|
||||
pointsNeeded,
|
||||
samplingFactor,
|
||||
);
|
||||
|
||||
return this.transformMetricsForGraph(
|
||||
completedMetrics,
|
||||
failedMetrics,
|
||||
timeRange,
|
||||
queueName,
|
||||
queueDetails,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error getting metrics for ${queueName}: ${error.message}`,
|
||||
);
|
||||
throw error;
|
||||
} finally {
|
||||
await queue.close();
|
||||
}
|
||||
}
|
||||
|
||||
private getPointsConfiguration(timeRange: QueueMetricsTimeRange): {
|
||||
pointsNeeded: number;
|
||||
samplingFactor: number;
|
||||
targetVisualizationPoints: number;
|
||||
} {
|
||||
const targetVisualizationPoints = 240;
|
||||
|
||||
let pointsNeeded: number;
|
||||
|
||||
switch (timeRange) {
|
||||
case QueueMetricsTimeRange.OneHour:
|
||||
pointsNeeded = 60; // 60 points (1 hour)
|
||||
break;
|
||||
case QueueMetricsTimeRange.FourHours:
|
||||
pointsNeeded = 4 * 60; // 240 points (4 hours)
|
||||
break;
|
||||
case QueueMetricsTimeRange.TwelveHours:
|
||||
pointsNeeded = 12 * 60; // 720 points (12 hours)
|
||||
break;
|
||||
case QueueMetricsTimeRange.OneDay:
|
||||
pointsNeeded = 24 * 60; // 1440 points (24 hours)
|
||||
break;
|
||||
case QueueMetricsTimeRange.SevenDays:
|
||||
pointsNeeded = 7 * 24 * 60; // 10080 points (7 days)
|
||||
break;
|
||||
|
||||
default:
|
||||
pointsNeeded = 24 * 60; // Default to 1 day
|
||||
}
|
||||
|
||||
const samplingFactor =
|
||||
pointsNeeded <= targetVisualizationPoints
|
||||
? 1
|
||||
: Math.ceil(pointsNeeded / targetVisualizationPoints);
|
||||
|
||||
return {
|
||||
pointsNeeded,
|
||||
samplingFactor,
|
||||
targetVisualizationPoints,
|
||||
};
|
||||
}
|
||||
|
||||
private extractMetricsData(
|
||||
metrics: number[] | undefined,
|
||||
pointsNeeded: number,
|
||||
samplingFactor = 1,
|
||||
): number[] {
|
||||
if (!metrics || !Array.isArray(metrics)) {
|
||||
return Array(Math.ceil(pointsNeeded / samplingFactor)).fill(0);
|
||||
}
|
||||
|
||||
try {
|
||||
const targetPoints = Math.ceil(pointsNeeded / samplingFactor);
|
||||
|
||||
const relevantData = metrics.slice(-pointsNeeded);
|
||||
const result: number[] = [];
|
||||
|
||||
const backfillCount = Math.max(
|
||||
0,
|
||||
targetPoints - Math.ceil(relevantData.length / samplingFactor),
|
||||
);
|
||||
|
||||
result.push(...Array(backfillCount).fill(0));
|
||||
|
||||
for (let i = 0; i < relevantData.length; i += samplingFactor) {
|
||||
const chunk = relevantData.slice(i, i + samplingFactor);
|
||||
|
||||
result.push(Math.max(...chunk));
|
||||
}
|
||||
|
||||
return result.slice(0, targetPoints);
|
||||
} catch (error) {
|
||||
this.logger.error(`Error extracting metrics data: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private transformMetricsForGraph(
|
||||
completedMetrics: number[],
|
||||
failedMetrics: number[],
|
||||
timeRange: QueueMetricsTimeRange,
|
||||
queueName: MessageQueue,
|
||||
queueDetails: WorkerQueueHealth | null,
|
||||
): QueueMetricsData {
|
||||
try {
|
||||
return {
|
||||
queueName,
|
||||
timeRange,
|
||||
details: queueDetails?.metrics ?? null,
|
||||
workers: queueDetails?.workers ?? 0,
|
||||
data: [
|
||||
{
|
||||
id: 'Completed Jobs',
|
||||
data: completedMetrics.map((count, index) => ({
|
||||
x: index,
|
||||
y: count,
|
||||
})),
|
||||
},
|
||||
{
|
||||
id: 'Failed Jobs',
|
||||
data: failedMetrics.map((count, index) => ({
|
||||
x: index,
|
||||
y: count,
|
||||
})),
|
||||
},
|
||||
],
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error transforming metrics for graph: ${error.message}`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,20 +10,24 @@ import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-he
|
||||
import { UpdateWorkspaceFeatureFlagInput } from 'src/engine/core-modules/admin-panel/dtos/update-workspace-feature-flag.input';
|
||||
import { UserLookup } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.entity';
|
||||
import { UserLookupInput } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.input';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-graphql-api-exception.filter';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { ImpersonateGuard } from 'src/engine/guards/impersonate-guard';
|
||||
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
|
||||
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
||||
|
||||
import { AdminPanelHealthServiceData } from './dtos/admin-panel-health-service-data.dto';
|
||||
|
||||
import { QueueMetricsData } from './dtos/queue-metrics-data.dto';
|
||||
@Resolver()
|
||||
@UseFilters(AuthGraphqlApiExceptionFilter)
|
||||
export class AdminPanelResolver {
|
||||
constructor(
|
||||
private adminService: AdminPanelService,
|
||||
private adminPanelHealthService: AdminPanelHealthService,
|
||||
private workerHealthIndicator: WorkerHealthIndicator,
|
||||
) {}
|
||||
|
||||
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard)
|
||||
@ -68,6 +72,7 @@ export class AdminPanelResolver {
|
||||
return this.adminPanelHealthService.getSystemHealthStatus();
|
||||
}
|
||||
|
||||
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard)
|
||||
@Query(() => AdminPanelHealthServiceData)
|
||||
async getIndicatorHealthStatus(
|
||||
@Args('indicatorId', {
|
||||
@ -77,4 +82,22 @@ export class AdminPanelResolver {
|
||||
): Promise<AdminPanelHealthServiceData> {
|
||||
return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorId);
|
||||
}
|
||||
|
||||
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard)
|
||||
@Query(() => QueueMetricsData)
|
||||
async getQueueMetrics(
|
||||
@Args('queueName', { type: () => String })
|
||||
queueName: string,
|
||||
@Args('timeRange', {
|
||||
nullable: true,
|
||||
defaultValue: QueueMetricsTimeRange.OneDay,
|
||||
type: () => QueueMetricsTimeRange,
|
||||
})
|
||||
timeRange: QueueMetricsTimeRange = QueueMetricsTimeRange.OneHour,
|
||||
): Promise<QueueMetricsData> {
|
||||
return await this.adminPanelHealthService.getQueueMetrics(
|
||||
queueName as MessageQueue,
|
||||
timeRange,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,13 +1,15 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
||||
|
||||
@ObjectType()
|
||||
export class AdminPanelWorkerQueueHealth extends WorkerQueueHealth {
|
||||
export class AdminPanelWorkerQueueHealth {
|
||||
@Field(() => String)
|
||||
id: string;
|
||||
|
||||
@Field(() => String)
|
||||
queueName: string;
|
||||
|
||||
@Field(() => AdminPanelHealthServiceStatus)
|
||||
status: AdminPanelHealthServiceStatus;
|
||||
}
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
@ObjectType()
|
||||
export class QueueMetricsDataPoint {
|
||||
@Field(() => Number)
|
||||
x: number;
|
||||
|
||||
@Field(() => Number)
|
||||
y: number;
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { QueueMetricsSeries } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-series.dto';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-queue-metrics.type';
|
||||
|
||||
@ObjectType()
|
||||
export class QueueMetricsData {
|
||||
@Field(() => String)
|
||||
queueName: string;
|
||||
|
||||
@Field(() => Number)
|
||||
workers: number;
|
||||
|
||||
@Field(() => QueueMetricsTimeRange)
|
||||
timeRange: QueueMetricsTimeRange;
|
||||
|
||||
@Field(() => WorkerQueueMetrics, { nullable: true })
|
||||
details: WorkerQueueMetrics | null;
|
||||
|
||||
@Field(() => [QueueMetricsSeries])
|
||||
data: QueueMetricsSeries[];
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { QueueMetricsDataPoint } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-data-point.dto';
|
||||
|
||||
@ObjectType()
|
||||
export class QueueMetricsSeries {
|
||||
@Field()
|
||||
id: string;
|
||||
|
||||
@Field(() => [QueueMetricsDataPoint])
|
||||
data: QueueMetricsDataPoint[];
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
import { registerEnumType } from '@nestjs/graphql';
|
||||
|
||||
export enum QueueMetricsTimeRange {
|
||||
SevenDays = '7D',
|
||||
OneDay = '1D',
|
||||
TwelveHours = '12H',
|
||||
FourHours = '4H',
|
||||
OneHour = '1H',
|
||||
}
|
||||
|
||||
registerEnumType(QueueMetricsTimeRange, {
|
||||
name: 'QueueMetricsTimeRange',
|
||||
});
|
||||
@ -12,12 +12,10 @@ import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-c
|
||||
const mockQueueInstance = {
|
||||
getWorkers: jest.fn().mockResolvedValue([]),
|
||||
close: jest.fn().mockResolvedValue(undefined),
|
||||
getFailedCount: jest.fn().mockResolvedValue(0),
|
||||
getCompletedCount: jest.fn().mockResolvedValue(0),
|
||||
getMetrics: jest.fn().mockResolvedValue({ count: 0, data: [] }),
|
||||
getWaitingCount: jest.fn().mockResolvedValue(0),
|
||||
getActiveCount: jest.fn().mockResolvedValue(0),
|
||||
getDelayedCount: jest.fn().mockResolvedValue(0),
|
||||
getPrioritizedCount: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
|
||||
jest.mock('bullmq', () => ({
|
||||
@ -28,6 +26,7 @@ describe('WorkerHealthIndicator', () => {
|
||||
let service: WorkerHealthIndicator;
|
||||
let mockRedis: jest.Mocked<Pick<Redis, 'ping'>>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
let loggerSpy: jest.SpyInstance;
|
||||
|
||||
beforeEach(async () => {
|
||||
mockRedis = {
|
||||
@ -64,11 +63,23 @@ describe('WorkerHealthIndicator', () => {
|
||||
}).compile();
|
||||
|
||||
service = module.get<WorkerHealthIndicator>(WorkerHealthIndicator);
|
||||
|
||||
loggerSpy = jest
|
||||
.spyOn(service['logger'], 'error')
|
||||
.mockImplementation(() => {});
|
||||
jest.useFakeTimers();
|
||||
|
||||
// Reset mocks to their default success state before each test
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([]);
|
||||
mockQueueInstance.getMetrics.mockResolvedValue({ count: 0, data: [] });
|
||||
mockQueueInstance.getWaitingCount.mockResolvedValue(0);
|
||||
mockQueueInstance.getActiveCount.mockResolvedValue(0);
|
||||
mockQueueInstance.getDelayedCount.mockResolvedValue(0);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
@ -133,4 +144,204 @@ describe('WorkerHealthIndicator', () => {
|
||||
Object.keys(MessageQueue).length,
|
||||
);
|
||||
});
|
||||
|
||||
it('should return down status when failure rate exceeds threshold', async () => {
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({ count: 600 });
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({ count: 400 });
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0 });
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('up');
|
||||
expect('queues' in result.worker).toBe(true);
|
||||
if ('queues' in result.worker) {
|
||||
expect(result.worker.queues[0].status).toBe('down');
|
||||
expect(result.worker.queues[0].metrics).toEqual({
|
||||
failed: 600,
|
||||
completed: 400,
|
||||
waiting: 0,
|
||||
active: 0,
|
||||
delayed: 0,
|
||||
failureRate: 60,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('should return complete metrics for active workers', async () => {
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({ count: 10 });
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({ count: 90 });
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0 });
|
||||
});
|
||||
mockQueueInstance.getWaitingCount.mockResolvedValue(5);
|
||||
mockQueueInstance.getActiveCount.mockResolvedValue(2);
|
||||
mockQueueInstance.getDelayedCount.mockResolvedValue(1);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('up');
|
||||
expect('queues' in result.worker).toBe(true);
|
||||
if ('queues' in result.worker) {
|
||||
expect(result.worker.queues[0].metrics).toEqual({
|
||||
failed: 10,
|
||||
completed: 90,
|
||||
waiting: 5,
|
||||
active: 2,
|
||||
delayed: 1,
|
||||
failureRate: 10,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle queue errors gracefully', async () => {
|
||||
mockQueueInstance.getWorkers.mockRejectedValue(new Error('Queue error'));
|
||||
mockQueueInstance.getMetrics.mockRejectedValue(new Error('Queue error'));
|
||||
mockQueueInstance.getWaitingCount.mockRejectedValue(
|
||||
new Error('Queue error'),
|
||||
);
|
||||
mockQueueInstance.getActiveCount.mockRejectedValue(
|
||||
new Error('Queue error'),
|
||||
);
|
||||
mockQueueInstance.getDelayedCount.mockRejectedValue(
|
||||
new Error('Queue error'),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('down');
|
||||
expect('error' in result.worker).toBe(true);
|
||||
if ('error' in result.worker) {
|
||||
expect(result.worker.error).toBe(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS);
|
||||
}
|
||||
|
||||
expect(loggerSpy).toHaveBeenCalled();
|
||||
Object.values(MessageQueue).forEach((queueName) => {
|
||||
expect(loggerSpy).toHaveBeenCalledWith(
|
||||
`Error getting queue details for ${queueName}: Queue error`,
|
||||
);
|
||||
expect(loggerSpy).toHaveBeenCalledWith(
|
||||
`Error checking worker for queue ${queueName}: Queue error`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getQueueDetails', () => {
|
||||
beforeEach(() => {
|
||||
// Reset mocks to clean state before each test in this describe block
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
mockQueueInstance.getMetrics.mockResolvedValue({ count: 0, data: [] });
|
||||
});
|
||||
|
||||
it('should return metrics with time series data when pointsNeeded is provided', async () => {
|
||||
const pointsNeeded = 60;
|
||||
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({
|
||||
count: 10,
|
||||
data: Array(pointsNeeded).fill(10 / pointsNeeded),
|
||||
});
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({
|
||||
count: 90,
|
||||
data: Array(pointsNeeded).fill(90 / pointsNeeded),
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0, data: [] });
|
||||
});
|
||||
|
||||
const result = await service.getQueueDetails(
|
||||
MessageQueue.messagingQueue,
|
||||
{
|
||||
pointsNeeded,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.metrics).toMatchObject({
|
||||
failed: 10,
|
||||
completed: 90,
|
||||
failedData: expect.any(Array),
|
||||
completedData: expect.any(Array),
|
||||
});
|
||||
expect(result?.metrics.failedData).toHaveLength(pointsNeeded);
|
||||
expect(result?.metrics.completedData).toHaveLength(pointsNeeded);
|
||||
});
|
||||
|
||||
it('should handle invalid metrics data gracefully', async () => {
|
||||
const invalidData = ['invalid', null, undefined, '1', 2];
|
||||
|
||||
mockQueueInstance.getMetrics.mockResolvedValue({
|
||||
count: 0,
|
||||
data: invalidData,
|
||||
});
|
||||
|
||||
const result = await service.getQueueDetails(
|
||||
MessageQueue.messagingQueue,
|
||||
{
|
||||
pointsNeeded: 5,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.metrics.failedData).toEqual([NaN, 0, NaN, 1, 2]);
|
||||
expect(result?.metrics.completedData).toEqual([NaN, 0, NaN, 1, 2]);
|
||||
});
|
||||
|
||||
it('should calculate correct failure rate with time series data', async () => {
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({ count: 600, data: Array(10).fill(60) });
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({ count: 400, data: Array(10).fill(40) });
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0, data: [] });
|
||||
});
|
||||
|
||||
const result = await service.getQueueDetails(
|
||||
MessageQueue.messagingQueue,
|
||||
{
|
||||
pointsNeeded: 10,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.metrics).toMatchObject({
|
||||
failed: 600,
|
||||
completed: 400,
|
||||
failureRate: 60,
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle queue errors gracefully', async () => {
|
||||
mockQueueInstance.getWorkers.mockRejectedValue(new Error('Queue error'));
|
||||
mockQueueInstance.getMetrics.mockRejectedValue(new Error('Queue error'));
|
||||
|
||||
await expect(
|
||||
service.getQueueDetails(MessageQueue.messagingQueue),
|
||||
).rejects.toThrow('Queue error');
|
||||
|
||||
expect(loggerSpy).toHaveBeenCalledWith(
|
||||
`Error getting queue details for ${MessageQueue.messagingQueue}: Queue error`,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import {
|
||||
HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const';
|
||||
import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
@ -11,12 +15,14 @@ import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-c
|
||||
|
||||
@Injectable()
|
||||
export class WorkerHealthIndicator {
|
||||
private readonly logger = new Logger(WorkerHealthIndicator.name);
|
||||
|
||||
constructor(
|
||||
private readonly redisClient: RedisClientService,
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
) {}
|
||||
|
||||
async isHealthy() {
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('worker');
|
||||
|
||||
try {
|
||||
@ -42,51 +48,106 @@ export class WorkerHealthIndicator {
|
||||
}
|
||||
}
|
||||
|
||||
private async checkWorkers() {
|
||||
async getQueueDetails(
|
||||
queueName: MessageQueue,
|
||||
options?: {
|
||||
pointsNeeded?: number;
|
||||
},
|
||||
): Promise<WorkerQueueHealth | null> {
|
||||
const redis = this.redisClient.getClient();
|
||||
const queue = new Queue(queueName, { connection: redis });
|
||||
|
||||
try {
|
||||
const workers = await queue.getWorkers();
|
||||
|
||||
if (workers.length > 0) {
|
||||
const metricsParams = options?.pointsNeeded
|
||||
? [0, options.pointsNeeded - 1]
|
||||
: [];
|
||||
|
||||
const [
|
||||
failedMetrics,
|
||||
completedMetrics,
|
||||
waitingCount,
|
||||
activeCount,
|
||||
delayedCount,
|
||||
] = await Promise.all([
|
||||
queue.getMetrics('failed', ...metricsParams),
|
||||
queue.getMetrics('completed', ...metricsParams),
|
||||
queue.getWaitingCount(),
|
||||
queue.getActiveCount(),
|
||||
queue.getDelayedCount(),
|
||||
]);
|
||||
|
||||
const failedCount = options?.pointsNeeded
|
||||
? this.calculateMetricsSum(failedMetrics.data)
|
||||
: failedMetrics.count;
|
||||
|
||||
const completedCount = options?.pointsNeeded
|
||||
? this.calculateMetricsSum(completedMetrics.data)
|
||||
: completedMetrics.count;
|
||||
|
||||
const totalJobs = failedCount + completedCount;
|
||||
const failureRate =
|
||||
totalJobs > 0
|
||||
? Number(((failedCount / totalJobs) * 100).toFixed(1))
|
||||
: 0;
|
||||
|
||||
return {
|
||||
queueName,
|
||||
workers: workers.length,
|
||||
status: failureRate > METRICS_FAILURE_RATE_THRESHOLD ? 'down' : 'up',
|
||||
metrics: {
|
||||
failed: failedCount,
|
||||
completed: completedCount,
|
||||
waiting: waitingCount,
|
||||
active: activeCount,
|
||||
delayed: delayedCount,
|
||||
failureRate,
|
||||
...(options?.pointsNeeded && {
|
||||
failedData: failedMetrics.data.map(Number),
|
||||
completedData: completedMetrics.data.map(Number),
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error getting queue details for ${queueName}: ${error.message}`,
|
||||
);
|
||||
throw error;
|
||||
} finally {
|
||||
await queue.close();
|
||||
}
|
||||
}
|
||||
|
||||
private calculateMetricsSum(data: string[] | number[]): number {
|
||||
const sum = data.reduce((sum: number, value: string | number) => {
|
||||
const numericValue = Number(value);
|
||||
|
||||
return sum + (isNaN(numericValue) ? 0 : numericValue);
|
||||
}, 0);
|
||||
|
||||
return Math.round(Number(sum));
|
||||
}
|
||||
|
||||
private async checkWorkers() {
|
||||
const queues = Object.values(MessageQueue);
|
||||
const queueStatuses: WorkerQueueHealth[] = [];
|
||||
|
||||
for (const queueName of queues) {
|
||||
const queue = new Queue(queueName, { connection: redis });
|
||||
|
||||
try {
|
||||
const workers = await queue.getWorkers();
|
||||
const queueDetails = await this.getQueueDetails(queueName);
|
||||
|
||||
if (workers.length > 0) {
|
||||
const [
|
||||
failedCount,
|
||||
completedCount,
|
||||
waitingCount,
|
||||
activeCount,
|
||||
delayedCount,
|
||||
prioritizedCount,
|
||||
] = await Promise.all([
|
||||
queue.getFailedCount(),
|
||||
queue.getCompletedCount(),
|
||||
queue.getWaitingCount(),
|
||||
queue.getActiveCount(),
|
||||
queue.getDelayedCount(),
|
||||
queue.getPrioritizedCount(),
|
||||
]);
|
||||
|
||||
queueStatuses.push({
|
||||
queueName: queueName,
|
||||
workers: workers.length,
|
||||
metrics: {
|
||||
failed: failedCount,
|
||||
completed: completedCount,
|
||||
waiting: waitingCount,
|
||||
active: activeCount,
|
||||
delayed: delayedCount,
|
||||
prioritized: prioritizedCount,
|
||||
},
|
||||
});
|
||||
if (queueDetails) {
|
||||
queueStatuses.push(queueDetails);
|
||||
}
|
||||
|
||||
await queue.close();
|
||||
} catch (error) {
|
||||
await queue.close();
|
||||
this.logger.error(
|
||||
`Error checking worker for queue ${queueName}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -7,6 +7,9 @@ export class WorkerQueueHealth {
|
||||
@Field(() => String)
|
||||
queueName: string;
|
||||
|
||||
@Field(() => String)
|
||||
status: string;
|
||||
|
||||
@Field(() => Number)
|
||||
workers: number;
|
||||
|
||||
|
||||
@ -18,5 +18,11 @@ export class WorkerQueueMetrics {
|
||||
delayed: number;
|
||||
|
||||
@Field(() => Number)
|
||||
prioritized: number;
|
||||
failureRate: number;
|
||||
|
||||
@Field(() => [Number], { nullable: true })
|
||||
failedData?: number[];
|
||||
|
||||
@Field(() => [Number], { nullable: true })
|
||||
completedData?: number[];
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { OnModuleDestroy } from '@nestjs/common';
|
||||
|
||||
import { JobsOptions, Queue, QueueOptions, Worker } from 'bullmq';
|
||||
import { JobsOptions, MetricsTime, Queue, QueueOptions, Worker } from 'bullmq';
|
||||
import { isDefined } from 'twenty-shared';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
@ -50,12 +50,16 @@ export class BullMQDriver implements MessageQueueDriver, OnModuleDestroy {
|
||||
handler: (job: MessageQueueJob<T>) => Promise<void>,
|
||||
options?: MessageQueueWorkerOptions,
|
||||
) {
|
||||
const workerOptions = isDefined(options?.concurrency)
|
||||
? {
|
||||
...this.options,
|
||||
concurrency: options.concurrency,
|
||||
}
|
||||
: this.options;
|
||||
const workerOptions = {
|
||||
...this.options,
|
||||
...(isDefined(options?.concurrency)
|
||||
? { concurrency: options.concurrency }
|
||||
: {}),
|
||||
metrics: {
|
||||
maxDataPoints: MetricsTime.ONE_WEEK,
|
||||
collectInterval: 60000,
|
||||
},
|
||||
};
|
||||
|
||||
this.workerMap[queueName] = new Worker(
|
||||
queueName,
|
||||
|
||||
Reference in New Issue
Block a user