Health status worker metrics improvements (#10442)
Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
@ -43,10 +43,8 @@ export enum AdminPanelHealthServiceStatus {
|
|||||||
export type AdminPanelWorkerQueueHealth = {
|
export type AdminPanelWorkerQueueHealth = {
|
||||||
__typename?: 'AdminPanelWorkerQueueHealth';
|
__typename?: 'AdminPanelWorkerQueueHealth';
|
||||||
id: Scalars['String'];
|
id: Scalars['String'];
|
||||||
metrics: WorkerQueueMetrics;
|
|
||||||
queueName: Scalars['String'];
|
queueName: Scalars['String'];
|
||||||
status: AdminPanelHealthServiceStatus;
|
status: AdminPanelHealthServiceStatus;
|
||||||
workers: Scalars['Float'];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export type Analytics = {
|
export type Analytics = {
|
||||||
@ -1310,6 +1308,7 @@ export type Query = {
|
|||||||
getPostgresCredentials?: Maybe<PostgresCredentials>;
|
getPostgresCredentials?: Maybe<PostgresCredentials>;
|
||||||
getProductPrices: BillingProductPricesOutput;
|
getProductPrices: BillingProductPricesOutput;
|
||||||
getPublicWorkspaceDataByDomain: PublicWorkspaceDataOutput;
|
getPublicWorkspaceDataByDomain: PublicWorkspaceDataOutput;
|
||||||
|
getQueueMetrics: QueueMetricsData;
|
||||||
getRoles: Array<Role>;
|
getRoles: Array<Role>;
|
||||||
getSSOIdentityProviders: Array<FindAvailableSsoidpOutput>;
|
getSSOIdentityProviders: Array<FindAvailableSsoidpOutput>;
|
||||||
getServerlessFunctionSourceCode?: Maybe<Scalars['JSON']>;
|
getServerlessFunctionSourceCode?: Maybe<Scalars['JSON']>;
|
||||||
@ -1369,6 +1368,12 @@ export type QueryGetProductPricesArgs = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
export type QueryGetQueueMetricsArgs = {
|
||||||
|
queueName: Scalars['String'];
|
||||||
|
timeRange?: InputMaybe<QueueMetricsTimeRange>;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
export type QueryGetServerlessFunctionSourceCodeArgs = {
|
export type QueryGetServerlessFunctionSourceCodeArgs = {
|
||||||
input: GetServerlessFunctionSourceCodeInput;
|
input: GetServerlessFunctionSourceCodeInput;
|
||||||
};
|
};
|
||||||
@ -1413,6 +1418,35 @@ export type QueryValidatePasswordResetTokenArgs = {
|
|||||||
passwordResetToken: Scalars['String'];
|
passwordResetToken: Scalars['String'];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type QueueMetricsData = {
|
||||||
|
__typename?: 'QueueMetricsData';
|
||||||
|
data: Array<QueueMetricsSeries>;
|
||||||
|
details?: Maybe<WorkerQueueMetrics>;
|
||||||
|
queueName: Scalars['String'];
|
||||||
|
timeRange: QueueMetricsTimeRange;
|
||||||
|
workers: Scalars['Float'];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type QueueMetricsDataPoint = {
|
||||||
|
__typename?: 'QueueMetricsDataPoint';
|
||||||
|
x: Scalars['Float'];
|
||||||
|
y: Scalars['Float'];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type QueueMetricsSeries = {
|
||||||
|
__typename?: 'QueueMetricsSeries';
|
||||||
|
data: Array<QueueMetricsDataPoint>;
|
||||||
|
id: Scalars['String'];
|
||||||
|
};
|
||||||
|
|
||||||
|
export enum QueueMetricsTimeRange {
|
||||||
|
FourHours = 'FourHours',
|
||||||
|
OneDay = 'OneDay',
|
||||||
|
OneHour = 'OneHour',
|
||||||
|
SevenDays = 'SevenDays',
|
||||||
|
TwelveHours = 'TwelveHours'
|
||||||
|
}
|
||||||
|
|
||||||
export type Relation = {
|
export type Relation = {
|
||||||
__typename?: 'Relation';
|
__typename?: 'Relation';
|
||||||
sourceFieldMetadata: Field;
|
sourceFieldMetadata: Field;
|
||||||
@ -1970,9 +2004,11 @@ export type WorkerQueueMetrics = {
|
|||||||
__typename?: 'WorkerQueueMetrics';
|
__typename?: 'WorkerQueueMetrics';
|
||||||
active: Scalars['Float'];
|
active: Scalars['Float'];
|
||||||
completed: Scalars['Float'];
|
completed: Scalars['Float'];
|
||||||
|
completedData?: Maybe<Array<Scalars['Float']>>;
|
||||||
delayed: Scalars['Float'];
|
delayed: Scalars['Float'];
|
||||||
failed: Scalars['Float'];
|
failed: Scalars['Float'];
|
||||||
prioritized: Scalars['Float'];
|
failedData?: Maybe<Array<Scalars['Float']>>;
|
||||||
|
failureRate: Scalars['Float'];
|
||||||
waiting: Scalars['Float'];
|
waiting: Scalars['Float'];
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -2394,7 +2430,15 @@ export type GetIndicatorHealthStatusQueryVariables = Exact<{
|
|||||||
}>;
|
}>;
|
||||||
|
|
||||||
|
|
||||||
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus, workers: number, metrics: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, prioritized: number } }> | null } };
|
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus }> | null } };
|
||||||
|
|
||||||
|
export type GetQueueMetricsQueryVariables = Exact<{
|
||||||
|
queueName: Scalars['String'];
|
||||||
|
timeRange?: InputMaybe<QueueMetricsTimeRange>;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
|
||||||
|
export type GetQueueMetricsQuery = { __typename?: 'Query', getQueueMetrics: { __typename?: 'QueueMetricsData', queueName: string, timeRange: QueueMetricsTimeRange, workers: number, details?: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, failureRate: number } | null, data: Array<{ __typename?: 'QueueMetricsSeries', id: string, data: Array<{ __typename?: 'QueueMetricsDataPoint', x: number, y: number }> }> } };
|
||||||
|
|
||||||
export type GetSystemHealthStatusQueryVariables = Exact<{ [key: string]: never; }>;
|
export type GetSystemHealthStatusQueryVariables = Exact<{ [key: string]: never; }>;
|
||||||
|
|
||||||
@ -4238,15 +4282,6 @@ export const GetIndicatorHealthStatusDocument = gql`
|
|||||||
id
|
id
|
||||||
queueName
|
queueName
|
||||||
status
|
status
|
||||||
workers
|
|
||||||
metrics {
|
|
||||||
failed
|
|
||||||
completed
|
|
||||||
waiting
|
|
||||||
active
|
|
||||||
delayed
|
|
||||||
prioritized
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -4279,6 +4314,59 @@ export function useGetIndicatorHealthStatusLazyQuery(baseOptions?: Apollo.LazyQu
|
|||||||
export type GetIndicatorHealthStatusQueryHookResult = ReturnType<typeof useGetIndicatorHealthStatusQuery>;
|
export type GetIndicatorHealthStatusQueryHookResult = ReturnType<typeof useGetIndicatorHealthStatusQuery>;
|
||||||
export type GetIndicatorHealthStatusLazyQueryHookResult = ReturnType<typeof useGetIndicatorHealthStatusLazyQuery>;
|
export type GetIndicatorHealthStatusLazyQueryHookResult = ReturnType<typeof useGetIndicatorHealthStatusLazyQuery>;
|
||||||
export type GetIndicatorHealthStatusQueryResult = Apollo.QueryResult<GetIndicatorHealthStatusQuery, GetIndicatorHealthStatusQueryVariables>;
|
export type GetIndicatorHealthStatusQueryResult = Apollo.QueryResult<GetIndicatorHealthStatusQuery, GetIndicatorHealthStatusQueryVariables>;
|
||||||
|
export const GetQueueMetricsDocument = gql`
|
||||||
|
query GetQueueMetrics($queueName: String!, $timeRange: QueueMetricsTimeRange) {
|
||||||
|
getQueueMetrics(queueName: $queueName, timeRange: $timeRange) {
|
||||||
|
queueName
|
||||||
|
timeRange
|
||||||
|
workers
|
||||||
|
details {
|
||||||
|
failed
|
||||||
|
completed
|
||||||
|
waiting
|
||||||
|
active
|
||||||
|
delayed
|
||||||
|
failureRate
|
||||||
|
}
|
||||||
|
data {
|
||||||
|
id
|
||||||
|
data {
|
||||||
|
x
|
||||||
|
y
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* __useGetQueueMetricsQuery__
|
||||||
|
*
|
||||||
|
* To run a query within a React component, call `useGetQueueMetricsQuery` and pass it any options that fit your needs.
|
||||||
|
* When your component renders, `useGetQueueMetricsQuery` returns an object from Apollo Client that contains loading, error, and data properties
|
||||||
|
* you can use to render your UI.
|
||||||
|
*
|
||||||
|
* @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options;
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* const { data, loading, error } = useGetQueueMetricsQuery({
|
||||||
|
* variables: {
|
||||||
|
* queueName: // value for 'queueName'
|
||||||
|
* timeRange: // value for 'timeRange'
|
||||||
|
* },
|
||||||
|
* });
|
||||||
|
*/
|
||||||
|
export function useGetQueueMetricsQuery(baseOptions: Apollo.QueryHookOptions<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>) {
|
||||||
|
const options = {...defaultOptions, ...baseOptions}
|
||||||
|
return Apollo.useQuery<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>(GetQueueMetricsDocument, options);
|
||||||
|
}
|
||||||
|
export function useGetQueueMetricsLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>) {
|
||||||
|
const options = {...defaultOptions, ...baseOptions}
|
||||||
|
return Apollo.useLazyQuery<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>(GetQueueMetricsDocument, options);
|
||||||
|
}
|
||||||
|
export type GetQueueMetricsQueryHookResult = ReturnType<typeof useGetQueueMetricsQuery>;
|
||||||
|
export type GetQueueMetricsLazyQueryHookResult = ReturnType<typeof useGetQueueMetricsLazyQuery>;
|
||||||
|
export type GetQueueMetricsQueryResult = Apollo.QueryResult<GetQueueMetricsQuery, GetQueueMetricsQueryVariables>;
|
||||||
export const GetSystemHealthStatusDocument = gql`
|
export const GetSystemHealthStatusDocument = gql`
|
||||||
query GetSystemHealthStatus {
|
query GetSystemHealthStatus {
|
||||||
getSystemHealthStatus {
|
getSystemHealthStatus {
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
import { SettingsAdminEnvVariablesTable } from '@/settings/admin-panel/components/SettingsAdminEnvVariablesTable';
|
import { SettingsAdminEnvVariablesTable } from '@/settings/admin-panel/components/SettingsAdminEnvVariablesTable';
|
||||||
|
import { SettingsAdminTabSkeletonLoader } from '@/settings/admin-panel/components/SettingsAdminTabSkeletonLoader';
|
||||||
import styled from '@emotion/styled';
|
import styled from '@emotion/styled';
|
||||||
import { useState } from 'react';
|
import { useState } from 'react';
|
||||||
import { Button, H1Title, H1TitleFontColor, Section } from 'twenty-ui';
|
import { Button, H1Title, H1TitleFontColor, Section } from 'twenty-ui';
|
||||||
@ -37,11 +38,10 @@ const StyledShowMoreButton = styled(Button)<{ isSelected?: boolean }>`
|
|||||||
`;
|
`;
|
||||||
|
|
||||||
export const SettingsAdminEnvVariables = () => {
|
export const SettingsAdminEnvVariables = () => {
|
||||||
const { data: environmentVariables } = useGetEnvironmentVariablesGroupedQuery(
|
const { data: environmentVariables, loading: environmentVariablesLoading } =
|
||||||
{
|
useGetEnvironmentVariablesGroupedQuery({
|
||||||
fetchPolicy: 'network-only',
|
fetchPolicy: 'network-only',
|
||||||
},
|
});
|
||||||
);
|
|
||||||
|
|
||||||
const [selectedGroup, setSelectedGroup] = useState<string | null>(null);
|
const [selectedGroup, setSelectedGroup] = useState<string | null>(null);
|
||||||
|
|
||||||
@ -64,6 +64,10 @@ export const SettingsAdminEnvVariables = () => {
|
|||||||
(group) => group.name === selectedGroup,
|
(group) => group.name === selectedGroup,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
if (environmentVariablesLoading) {
|
||||||
|
return <SettingsAdminTabSkeletonLoader />;
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<Section>
|
<Section>
|
||||||
|
|||||||
@ -0,0 +1,17 @@
|
|||||||
|
import { useTheme } from '@emotion/react';
|
||||||
|
import Skeleton, { SkeletonTheme } from 'react-loading-skeleton';
|
||||||
|
|
||||||
|
import { SKELETON_LOADER_HEIGHT_SIZES } from '@/activities/components/SkeletonLoader';
|
||||||
|
|
||||||
|
export const SettingsAdminTabSkeletonLoader = () => {
|
||||||
|
const theme = useTheme();
|
||||||
|
return (
|
||||||
|
<SkeletonTheme
|
||||||
|
baseColor={theme.background.tertiary}
|
||||||
|
highlightColor={theme.background.transparent.lighter}
|
||||||
|
borderRadius={4}
|
||||||
|
>
|
||||||
|
<Skeleton height={SKELETON_LOADER_HEIGHT_SIZES.standard.m} width={120} />
|
||||||
|
</SkeletonTheme>
|
||||||
|
);
|
||||||
|
};
|
||||||
@ -19,9 +19,7 @@ const StyledErrorMessage = styled.div`
|
|||||||
`;
|
`;
|
||||||
|
|
||||||
export const DatabaseAndRedisHealthStatus = () => {
|
export const DatabaseAndRedisHealthStatus = () => {
|
||||||
const { indicatorHealth, loading } = useContext(
|
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
|
||||||
SettingsAdminIndicatorHealthContext,
|
|
||||||
);
|
|
||||||
|
|
||||||
const formattedDetails = indicatorHealth.details
|
const formattedDetails = indicatorHealth.details
|
||||||
? JSON.stringify(JSON.parse(indicatorHealth.details), null, 2)
|
? JSON.stringify(JSON.parse(indicatorHealth.details), null, 2)
|
||||||
@ -33,7 +31,7 @@ export const DatabaseAndRedisHealthStatus = () => {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<Section>
|
<Section>
|
||||||
{isDatabaseOrRedisDown && !loading ? (
|
{isDatabaseOrRedisDown ? (
|
||||||
<StyledErrorMessage>
|
<StyledErrorMessage>
|
||||||
{`${indicatorHealth.label} information is not available because the service is down`}
|
{`${indicatorHealth.label} information is not available because the service is down`}
|
||||||
</StyledErrorMessage>
|
</StyledErrorMessage>
|
||||||
|
|||||||
@ -1,18 +1,27 @@
|
|||||||
|
import { SettingsAdminTabSkeletonLoader } from '@/settings/admin-panel/components/SettingsAdminTabSkeletonLoader';
|
||||||
import { SettingsHealthStatusListCard } from '@/settings/admin-panel/health-status/components/SettingsHealthStatusListCard';
|
import { SettingsHealthStatusListCard } from '@/settings/admin-panel/health-status/components/SettingsHealthStatusListCard';
|
||||||
import { H2Title, Section } from 'twenty-ui';
|
import { H2Title, Section } from 'twenty-ui';
|
||||||
import { useGetSystemHealthStatusQuery } from '~/generated/graphql';
|
import { useGetSystemHealthStatusQuery } from '~/generated/graphql';
|
||||||
|
|
||||||
export const SettingsAdminHealthStatus = () => {
|
export const SettingsAdminHealthStatus = () => {
|
||||||
const { data, loading } = useGetSystemHealthStatusQuery({
|
const { data, loading: loadingHealthStatus } = useGetSystemHealthStatusQuery({
|
||||||
fetchPolicy: 'network-only',
|
fetchPolicy: 'network-only',
|
||||||
});
|
});
|
||||||
|
|
||||||
const services = data?.getSystemHealthStatus.services ?? [];
|
const services = data?.getSystemHealthStatus.services ?? [];
|
||||||
|
|
||||||
|
if (loadingHealthStatus) {
|
||||||
|
return <SettingsAdminTabSkeletonLoader />;
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<Section>
|
<Section>
|
||||||
<H2Title title="Health Status" description="How your system is doing" />
|
<H2Title title="Health Status" description="How your system is doing" />
|
||||||
<SettingsHealthStatusListCard services={services} loading={loading} />
|
<SettingsHealthStatusListCard
|
||||||
|
services={services}
|
||||||
|
loading={loadingHealthStatus}
|
||||||
|
/>
|
||||||
</Section>
|
</Section>
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
|
|||||||
@ -1,105 +0,0 @@
|
|||||||
import { SettingsListCard } from '@/settings/components/SettingsListCard';
|
|
||||||
import { Table } from '@/ui/layout/table/components/Table';
|
|
||||||
import { TableCell } from '@/ui/layout/table/components/TableCell';
|
|
||||||
import { TableRow } from '@/ui/layout/table/components/TableRow';
|
|
||||||
import styled from '@emotion/styled';
|
|
||||||
import { AnimatedExpandableContainer, Status } from 'twenty-ui';
|
|
||||||
import {
|
|
||||||
AdminPanelHealthServiceStatus,
|
|
||||||
AdminPanelWorkerQueueHealth,
|
|
||||||
} from '~/generated/graphql';
|
|
||||||
|
|
||||||
const StyledExpandedContent = styled.div`
|
|
||||||
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
|
||||||
border-radius: ${({ theme }) => theme.border.radius.sm};
|
|
||||||
padding-top: ${({ theme }) => theme.spacing(1)};
|
|
||||||
padding-bottom: ${({ theme }) => theme.spacing(3)};
|
|
||||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
|
||||||
padding-right: ${({ theme }) => theme.spacing(3)};
|
|
||||||
`;
|
|
||||||
|
|
||||||
const StyledContainer = styled.div`
|
|
||||||
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
|
||||||
margin-top: ${({ theme }) => theme.spacing(5)};
|
|
||||||
`;
|
|
||||||
|
|
||||||
const StyledTableRow = styled(TableRow)`
|
|
||||||
height: ${({ theme }) => theme.spacing(6)};
|
|
||||||
`;
|
|
||||||
|
|
||||||
const StyledQueueMetricsTitle = styled.div`
|
|
||||||
color: ${({ theme }) => theme.font.color.primary};
|
|
||||||
font-size: ${({ theme }) => theme.font.size.sm};
|
|
||||||
font-weight: ${({ theme }) => theme.font.weight.medium};
|
|
||||||
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
|
||||||
padding-left: ${({ theme }) => theme.spacing(3)};
|
|
||||||
`;
|
|
||||||
|
|
||||||
export const SettingsAdminQueueExpandableContainer = ({
|
|
||||||
queues,
|
|
||||||
selectedQueue,
|
|
||||||
}: {
|
|
||||||
queues: AdminPanelWorkerQueueHealth[];
|
|
||||||
selectedQueue: string | null;
|
|
||||||
}) => {
|
|
||||||
const selectedQueueData = queues.find(
|
|
||||||
(queue) => queue.queueName === selectedQueue,
|
|
||||||
);
|
|
||||||
|
|
||||||
return (
|
|
||||||
<AnimatedExpandableContainer
|
|
||||||
isExpanded={!!selectedQueue}
|
|
||||||
mode="fit-content"
|
|
||||||
>
|
|
||||||
{selectedQueueData && (
|
|
||||||
<>
|
|
||||||
<StyledContainer>
|
|
||||||
<SettingsListCard
|
|
||||||
items={[
|
|
||||||
{ ...selectedQueueData, id: selectedQueueData.queueName },
|
|
||||||
]}
|
|
||||||
getItemLabel={(
|
|
||||||
item: AdminPanelWorkerQueueHealth & { id: string },
|
|
||||||
) => item.queueName}
|
|
||||||
isLoading={false}
|
|
||||||
RowRightComponent={({
|
|
||||||
item,
|
|
||||||
}: {
|
|
||||||
item: AdminPanelWorkerQueueHealth;
|
|
||||||
}) => (
|
|
||||||
<Status
|
|
||||||
color={
|
|
||||||
item.status === AdminPanelHealthServiceStatus.OPERATIONAL
|
|
||||||
? 'green'
|
|
||||||
: 'red'
|
|
||||||
}
|
|
||||||
text={item.status.toLowerCase()}
|
|
||||||
weight="medium"
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
/>
|
|
||||||
</StyledContainer>
|
|
||||||
<StyledQueueMetricsTitle> Metrics:</StyledQueueMetricsTitle>
|
|
||||||
<StyledExpandedContent>
|
|
||||||
<Table>
|
|
||||||
<StyledTableRow>
|
|
||||||
<TableCell align="left">Workers</TableCell>
|
|
||||||
<TableCell align="right">{selectedQueueData.workers}</TableCell>
|
|
||||||
</StyledTableRow>
|
|
||||||
{Object.entries(selectedQueueData.metrics)
|
|
||||||
.filter(([key]) => key !== '__typename')
|
|
||||||
.map(([key, value]) => (
|
|
||||||
<StyledTableRow key={key}>
|
|
||||||
<TableCell align="left">
|
|
||||||
{key.charAt(0).toUpperCase() + key.slice(1)}
|
|
||||||
</TableCell>
|
|
||||||
<TableCell align="right">{value}</TableCell>
|
|
||||||
</StyledTableRow>
|
|
||||||
))}
|
|
||||||
</Table>
|
|
||||||
</StyledExpandedContent>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</AnimatedExpandableContainer>
|
|
||||||
);
|
|
||||||
};
|
|
||||||
@ -1,52 +0,0 @@
|
|||||||
import styled from '@emotion/styled';
|
|
||||||
import { Button } from 'twenty-ui';
|
|
||||||
import {
|
|
||||||
AdminPanelHealthServiceStatus,
|
|
||||||
AdminPanelWorkerQueueHealth,
|
|
||||||
} from '~/generated/graphql';
|
|
||||||
|
|
||||||
const StyledQueueButtonsRow = styled.div`
|
|
||||||
display: flex;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: ${({ theme }) => theme.spacing(2)};
|
|
||||||
margin-top: ${({ theme }) => theme.spacing(6)};
|
|
||||||
`;
|
|
||||||
|
|
||||||
const StyledQueueHealthButton = styled(Button)<{
|
|
||||||
isSelected?: boolean;
|
|
||||||
status: AdminPanelHealthServiceStatus;
|
|
||||||
}>`
|
|
||||||
${({ isSelected, theme, status }) =>
|
|
||||||
isSelected &&
|
|
||||||
`
|
|
||||||
background-color: ${
|
|
||||||
status === AdminPanelHealthServiceStatus.OPERATIONAL
|
|
||||||
? theme.tag.background.green
|
|
||||||
: theme.tag.background.red
|
|
||||||
};
|
|
||||||
`}
|
|
||||||
`;
|
|
||||||
export const SettingsAdminQueueHealthButtons = ({
|
|
||||||
queues,
|
|
||||||
selectedQueue,
|
|
||||||
toggleQueueVisibility,
|
|
||||||
}: {
|
|
||||||
queues: AdminPanelWorkerQueueHealth[];
|
|
||||||
selectedQueue: string | null;
|
|
||||||
toggleQueueVisibility: (queueName: string) => void;
|
|
||||||
}) => {
|
|
||||||
return (
|
|
||||||
<StyledQueueButtonsRow>
|
|
||||||
{queues.map((queue) => (
|
|
||||||
<StyledQueueHealthButton
|
|
||||||
key={queue.queueName}
|
|
||||||
onClick={() => toggleQueueVisibility(queue.queueName)}
|
|
||||||
title={queue.queueName}
|
|
||||||
variant="secondary"
|
|
||||||
isSelected={selectedQueue === queue.queueName}
|
|
||||||
status={queue.status}
|
|
||||||
/>
|
|
||||||
))}
|
|
||||||
</StyledQueueButtonsRow>
|
|
||||||
);
|
|
||||||
};
|
|
||||||
@ -1,16 +1,8 @@
|
|||||||
import { SettingsAdminQueueExpandableContainer } from '@/settings/admin-panel/health-status/components/SettingsAdminQueueExpandableContainer';
|
import { WorkerQueueMetricsSection } from '@/settings/admin-panel/health-status/components/WorkerQueueMetricsSection';
|
||||||
import { SettingsAdminQueueHealthButtons } from '@/settings/admin-panel/health-status/components/SettingsAdminQueueHealthButtons';
|
|
||||||
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
|
|
||||||
import styled from '@emotion/styled';
|
import styled from '@emotion/styled';
|
||||||
import { useContext, useState } from 'react';
|
import { useContext } from 'react';
|
||||||
import { H2Title, Section } from 'twenty-ui';
|
|
||||||
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
|
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
|
||||||
|
import { SettingsAdminIndicatorHealthContext } from '../contexts/SettingsAdminIndicatorHealthContext';
|
||||||
const StyledTitleContainer = styled.div`
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
flex: 1;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const StyledErrorMessage = styled.div`
|
const StyledErrorMessage = styled.div`
|
||||||
color: ${({ theme }) => theme.color.red};
|
color: ${({ theme }) => theme.color.red};
|
||||||
@ -18,45 +10,23 @@ const StyledErrorMessage = styled.div`
|
|||||||
`;
|
`;
|
||||||
|
|
||||||
export const WorkerHealthStatus = () => {
|
export const WorkerHealthStatus = () => {
|
||||||
const { indicatorHealth, loading } = useContext(
|
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
|
||||||
SettingsAdminIndicatorHealthContext,
|
|
||||||
);
|
|
||||||
|
|
||||||
const isWorkerDown =
|
const isWorkerDown =
|
||||||
!indicatorHealth.status ||
|
!indicatorHealth.status ||
|
||||||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
|
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
|
||||||
|
|
||||||
const [selectedQueue, setSelectedQueue] = useState<string | null>(null);
|
|
||||||
|
|
||||||
const toggleQueueVisibility = (queueName: string) => {
|
|
||||||
setSelectedQueue(selectedQueue === queueName ? null : queueName);
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Section>
|
<>
|
||||||
<StyledTitleContainer>
|
{isWorkerDown ? (
|
||||||
<H2Title
|
|
||||||
title="Queue Status"
|
|
||||||
description="Background job processing status and metrics"
|
|
||||||
/>
|
|
||||||
</StyledTitleContainer>
|
|
||||||
{isWorkerDown && !loading ? (
|
|
||||||
<StyledErrorMessage>
|
<StyledErrorMessage>
|
||||||
Queue information is not available because the worker is down
|
Queue information is not available because the worker is down
|
||||||
</StyledErrorMessage>
|
</StyledErrorMessage>
|
||||||
) : (
|
) : (
|
||||||
<>
|
(indicatorHealth.queues ?? []).map((queue) => (
|
||||||
<SettingsAdminQueueHealthButtons
|
<WorkerQueueMetricsSection key={queue.queueName} queue={queue} />
|
||||||
queues={indicatorHealth.queues ?? []}
|
))
|
||||||
selectedQueue={selectedQueue}
|
|
||||||
toggleQueueVisibility={toggleQueueVisibility}
|
|
||||||
/>
|
|
||||||
<SettingsAdminQueueExpandableContainer
|
|
||||||
queues={indicatorHealth.queues ?? []}
|
|
||||||
selectedQueue={selectedQueue}
|
|
||||||
/>
|
|
||||||
</>
|
|
||||||
)}
|
)}
|
||||||
</Section>
|
</>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|||||||
@ -0,0 +1,315 @@
|
|||||||
|
import { useTheme } from '@emotion/react';
|
||||||
|
import styled from '@emotion/styled';
|
||||||
|
import { ResponsiveLine } from '@nivo/line';
|
||||||
|
|
||||||
|
import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar';
|
||||||
|
import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
|
||||||
|
import { Select } from '@/ui/input/components/Select';
|
||||||
|
import { Table } from '@/ui/layout/table/components/Table';
|
||||||
|
import { TableCell } from '@/ui/layout/table/components/TableCell';
|
||||||
|
import { TableRow } from '@/ui/layout/table/components/TableRow';
|
||||||
|
import {
|
||||||
|
QueueMetricsTimeRange,
|
||||||
|
useGetQueueMetricsQuery,
|
||||||
|
} from '~/generated/graphql';
|
||||||
|
|
||||||
|
const StyledTableRow = styled(TableRow)`
|
||||||
|
height: ${({ theme }) => theme.spacing(6)};
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledQueueMetricsTitle = styled.div`
|
||||||
|
color: ${({ theme }) => theme.font.color.primary};
|
||||||
|
font-size: ${({ theme }) => theme.font.size.sm};
|
||||||
|
font-weight: ${({ theme }) => theme.font.weight.medium};
|
||||||
|
margin-bottom: ${({ theme }) => theme.spacing(3)};
|
||||||
|
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledGraphContainer = styled.div`
|
||||||
|
background-color: ${({ theme }) => theme.background.tertiary};
|
||||||
|
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||||
|
height: 230px;
|
||||||
|
margin-bottom: ${({ theme }) => theme.spacing(4)};
|
||||||
|
padding-top: ${({ theme }) => theme.spacing(4)};
|
||||||
|
padding-bottom: ${({ theme }) => theme.spacing(2)};
|
||||||
|
width: 100%;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledQueueMetricsContainer = styled.div`
|
||||||
|
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||||
|
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||||
|
padding-top: ${({ theme }) => theme.spacing(1)};
|
||||||
|
padding-bottom: ${({ theme }) => theme.spacing(3)};
|
||||||
|
padding-left: ${({ theme }) => theme.spacing(3)};
|
||||||
|
padding-right: ${({ theme }) => theme.spacing(3)};
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledGraphControls = styled.div`
|
||||||
|
display: flex;
|
||||||
|
gap: ${({ theme }) => theme.spacing(2)};
|
||||||
|
justify-content: flex-end;
|
||||||
|
margin-bottom: ${({ theme }) => theme.spacing(2)};
|
||||||
|
margin-top: ${({ theme }) => theme.spacing(1)};
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledNoDataMessage = styled.div`
|
||||||
|
align-items: center;
|
||||||
|
color: ${({ theme }) => theme.font.color.light};
|
||||||
|
display: flex;
|
||||||
|
height: 100%;
|
||||||
|
justify-content: center;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledTooltipContainer = styled.div`
|
||||||
|
background-color: ${({ theme }) => theme.background.secondary};
|
||||||
|
border: 1px solid ${({ theme }) => theme.border.color.medium};
|
||||||
|
border-radius: ${({ theme }) => theme.border.radius.sm};
|
||||||
|
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
||||||
|
padding: ${({ theme }) => theme.spacing(2)};
|
||||||
|
font-size: ${({ theme }) => theme.font.size.sm};
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledTooltipItem = styled.div<{ color: string }>`
|
||||||
|
align-items: center;
|
||||||
|
color: ${({ theme }) => theme.font.color.primary};
|
||||||
|
display: flex;
|
||||||
|
padding: ${({ theme }) => theme.spacing(0.5)} 0;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledTooltipColorSquare = styled.div<{ color: string }>`
|
||||||
|
width: 12px;
|
||||||
|
height: 12px;
|
||||||
|
background-color: ${({ color }) => color};
|
||||||
|
margin-right: ${({ theme }) => theme.spacing(1)};
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledTooltipValue = styled.span`
|
||||||
|
font-weight: ${({ theme }) => theme.font.weight.medium};
|
||||||
|
`;
|
||||||
|
|
||||||
|
type WorkerMetricsGraphProps = {
|
||||||
|
queueName: string;
|
||||||
|
timeRange: QueueMetricsTimeRange;
|
||||||
|
onTimeRangeChange: (range: QueueMetricsTimeRange) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const WorkerMetricsGraph = ({
|
||||||
|
queueName,
|
||||||
|
timeRange,
|
||||||
|
onTimeRangeChange,
|
||||||
|
}: WorkerMetricsGraphProps) => {
|
||||||
|
const theme = useTheme();
|
||||||
|
const { enqueueSnackBar } = useSnackBar();
|
||||||
|
|
||||||
|
const { loading, data } = useGetQueueMetricsQuery({
|
||||||
|
variables: {
|
||||||
|
queueName,
|
||||||
|
timeRange,
|
||||||
|
},
|
||||||
|
fetchPolicy: 'no-cache',
|
||||||
|
onError: (error) => {
|
||||||
|
enqueueSnackBar(`Error fetching worker metrics: ${error.message}`, {
|
||||||
|
variant: SnackBarVariant.Error,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const metricsData = data?.getQueueMetrics?.data || [];
|
||||||
|
const hasData =
|
||||||
|
metricsData.length > 0 &&
|
||||||
|
metricsData.some((series) => series.data.length > 0);
|
||||||
|
|
||||||
|
const metricsDetails = {
|
||||||
|
workers: data?.getQueueMetrics?.workers,
|
||||||
|
...data?.getQueueMetrics?.details,
|
||||||
|
};
|
||||||
|
|
||||||
|
const getMaxYValue = () => {
|
||||||
|
if (!hasData) return 2;
|
||||||
|
|
||||||
|
let maxValue = 0;
|
||||||
|
metricsData.forEach((series) => {
|
||||||
|
series.data.forEach((point) => {
|
||||||
|
if (typeof point.y === 'number' && point.y > maxValue) {
|
||||||
|
maxValue = point.y;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return maxValue === 0 ? 2 : maxValue * 1.1;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getAxisLabel = () => {
|
||||||
|
switch (timeRange) {
|
||||||
|
case QueueMetricsTimeRange.OneHour:
|
||||||
|
return 'Last 1 Hour (oldest → newest)';
|
||||||
|
case QueueMetricsTimeRange.FourHours:
|
||||||
|
return 'Last 4 Hours (oldest → newest)';
|
||||||
|
case QueueMetricsTimeRange.TwelveHours:
|
||||||
|
return 'Last 12 Hours (oldest → newest)';
|
||||||
|
case QueueMetricsTimeRange.OneDay:
|
||||||
|
return 'Last 24 Hours (oldest → newest)';
|
||||||
|
case QueueMetricsTimeRange.SevenDays:
|
||||||
|
return 'Last 7 Days (oldest → newest)';
|
||||||
|
default:
|
||||||
|
return 'Recent Events (oldest → newest)';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<StyledGraphControls>
|
||||||
|
<Select
|
||||||
|
dropdownId={`timerange-${queueName}`}
|
||||||
|
value={timeRange}
|
||||||
|
options={[
|
||||||
|
{ value: QueueMetricsTimeRange.SevenDays, label: 'This week' },
|
||||||
|
{ value: QueueMetricsTimeRange.OneDay, label: 'Today' },
|
||||||
|
{
|
||||||
|
value: QueueMetricsTimeRange.TwelveHours,
|
||||||
|
label: 'Last 12 hours',
|
||||||
|
},
|
||||||
|
{ value: QueueMetricsTimeRange.FourHours, label: 'Last 4 hours' },
|
||||||
|
{ value: QueueMetricsTimeRange.OneHour, label: 'Last 1 hour' },
|
||||||
|
]}
|
||||||
|
onChange={onTimeRangeChange}
|
||||||
|
needIconCheck
|
||||||
|
/>
|
||||||
|
</StyledGraphControls>
|
||||||
|
|
||||||
|
<StyledGraphContainer>
|
||||||
|
{loading ? (
|
||||||
|
<StyledNoDataMessage>Loading metrics data...</StyledNoDataMessage>
|
||||||
|
) : hasData ? (
|
||||||
|
<ResponsiveLine
|
||||||
|
data={metricsData}
|
||||||
|
curve="monotoneX"
|
||||||
|
enableArea={true}
|
||||||
|
colors={[theme.color.green, theme.color.red]}
|
||||||
|
theme={{
|
||||||
|
text: {
|
||||||
|
fill: theme.font.color.light,
|
||||||
|
fontSize: theme.font.size.sm,
|
||||||
|
fontFamily: theme.font.family,
|
||||||
|
},
|
||||||
|
axis: {
|
||||||
|
domain: {
|
||||||
|
line: {
|
||||||
|
stroke: theme.border.color.strong,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ticks: {
|
||||||
|
line: {
|
||||||
|
stroke: theme.border.color.strong,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
grid: {
|
||||||
|
line: {
|
||||||
|
stroke: theme.border.color.medium,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
crosshair: {
|
||||||
|
line: {
|
||||||
|
stroke: theme.font.color.primary,
|
||||||
|
strokeDasharray: '2 2',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
margin={{ top: 40, right: 30, bottom: 40, left: 50 }}
|
||||||
|
xScale={{
|
||||||
|
type: 'linear',
|
||||||
|
min: 0,
|
||||||
|
max: 'auto',
|
||||||
|
}}
|
||||||
|
yScale={{
|
||||||
|
type: 'linear',
|
||||||
|
min: 0,
|
||||||
|
max: getMaxYValue(),
|
||||||
|
stacked: false,
|
||||||
|
}}
|
||||||
|
axisBottom={{
|
||||||
|
legend: getAxisLabel(),
|
||||||
|
legendOffset: 30,
|
||||||
|
legendPosition: 'middle',
|
||||||
|
tickSize: 5,
|
||||||
|
tickPadding: 5,
|
||||||
|
tickValues: 5,
|
||||||
|
format: () => '',
|
||||||
|
}}
|
||||||
|
axisLeft={{
|
||||||
|
tickSize: 6,
|
||||||
|
tickPadding: 5,
|
||||||
|
tickValues: 4,
|
||||||
|
legend: 'Count',
|
||||||
|
legendOffset: -40,
|
||||||
|
legendPosition: 'middle',
|
||||||
|
}}
|
||||||
|
enableGridX={false}
|
||||||
|
gridYValues={4}
|
||||||
|
pointSize={0}
|
||||||
|
enableSlices="x"
|
||||||
|
sliceTooltip={({ slice }) => (
|
||||||
|
<StyledTooltipContainer>
|
||||||
|
{slice.points.map((point) => (
|
||||||
|
<StyledTooltipItem key={point.id} color={point.serieColor}>
|
||||||
|
<StyledTooltipColorSquare color={point.serieColor} />
|
||||||
|
<span>
|
||||||
|
{point.serieId}:{' '}
|
||||||
|
<StyledTooltipValue>
|
||||||
|
{String(point.data.y)}
|
||||||
|
</StyledTooltipValue>
|
||||||
|
</span>
|
||||||
|
</StyledTooltipItem>
|
||||||
|
))}
|
||||||
|
</StyledTooltipContainer>
|
||||||
|
)}
|
||||||
|
useMesh={true}
|
||||||
|
legends={[
|
||||||
|
{
|
||||||
|
anchor: 'top-right',
|
||||||
|
direction: 'row',
|
||||||
|
justify: false,
|
||||||
|
translateX: 0,
|
||||||
|
translateY: -40,
|
||||||
|
itemsSpacing: 10,
|
||||||
|
itemDirection: 'left-to-right',
|
||||||
|
itemWidth: 100,
|
||||||
|
itemHeight: 20,
|
||||||
|
symbolSize: 12,
|
||||||
|
symbolShape: 'square',
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<StyledNoDataMessage>No metrics data available</StyledNoDataMessage>
|
||||||
|
)}
|
||||||
|
</StyledGraphContainer>
|
||||||
|
{metricsDetails && (
|
||||||
|
<>
|
||||||
|
<StyledQueueMetricsTitle>Metrics:</StyledQueueMetricsTitle>
|
||||||
|
<StyledQueueMetricsContainer>
|
||||||
|
<Table>
|
||||||
|
{Object.entries(metricsDetails)
|
||||||
|
.filter(([key]) => key !== '__typename')
|
||||||
|
.map(([key, value]) => (
|
||||||
|
<StyledTableRow key={key}>
|
||||||
|
<TableCell align="left">
|
||||||
|
{key.charAt(0).toUpperCase() + key.slice(1)}
|
||||||
|
</TableCell>
|
||||||
|
<TableCell align="right">
|
||||||
|
{typeof value === 'number'
|
||||||
|
? value
|
||||||
|
: Array.isArray(value)
|
||||||
|
? value.length
|
||||||
|
: String(value)}
|
||||||
|
</TableCell>
|
||||||
|
</StyledTableRow>
|
||||||
|
))}
|
||||||
|
</Table>
|
||||||
|
</StyledQueueMetricsContainer>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { H2Title, Section } from 'twenty-ui';
|
||||||
|
import {
|
||||||
|
AdminPanelWorkerQueueHealth,
|
||||||
|
QueueMetricsTimeRange,
|
||||||
|
} from '~/generated/graphql';
|
||||||
|
import { WorkerMetricsGraph } from './WorkerMetricsGraph';
|
||||||
|
|
||||||
|
type WorkerQueueMetricsSectionProps = {
|
||||||
|
queue: AdminPanelWorkerQueueHealth;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const WorkerQueueMetricsSection = ({
|
||||||
|
queue,
|
||||||
|
}: WorkerQueueMetricsSectionProps) => {
|
||||||
|
const [timeRange, setTimeRange] = useState(QueueMetricsTimeRange.OneHour);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Section>
|
||||||
|
<H2Title title={queue.queueName} description="Queue performance" />
|
||||||
|
<WorkerMetricsGraph
|
||||||
|
queueName={queue.queueName}
|
||||||
|
timeRange={timeRange}
|
||||||
|
onTimeRangeChange={setTimeRange}
|
||||||
|
/>
|
||||||
|
</Section>
|
||||||
|
);
|
||||||
|
};
|
||||||
@ -6,7 +6,6 @@ import {
|
|||||||
|
|
||||||
type SettingsAdminIndicatorHealthContextType = {
|
type SettingsAdminIndicatorHealthContextType = {
|
||||||
indicatorHealth: AdminPanelHealthServiceData;
|
indicatorHealth: AdminPanelHealthServiceData;
|
||||||
loading: boolean;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export const SettingsAdminIndicatorHealthContext =
|
export const SettingsAdminIndicatorHealthContext =
|
||||||
@ -19,5 +18,4 @@ export const SettingsAdminIndicatorHealthContext =
|
|||||||
details: '',
|
details: '',
|
||||||
queues: [],
|
queues: [],
|
||||||
},
|
},
|
||||||
loading: false,
|
|
||||||
});
|
});
|
||||||
|
|||||||
@ -12,15 +12,6 @@ export const GET_INDICATOR_HEALTH_STATUS = gql`
|
|||||||
id
|
id
|
||||||
queueName
|
queueName
|
||||||
status
|
status
|
||||||
workers
|
|
||||||
metrics {
|
|
||||||
failed
|
|
||||||
completed
|
|
||||||
waiting
|
|
||||||
active
|
|
||||||
delayed
|
|
||||||
prioritized
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,29 @@
|
|||||||
|
import { gql } from '@apollo/client';
|
||||||
|
|
||||||
|
export const GET_QUEUE_METRICS = gql`
|
||||||
|
query GetQueueMetrics(
|
||||||
|
$queueName: String!
|
||||||
|
$timeRange: QueueMetricsTimeRange
|
||||||
|
) {
|
||||||
|
getQueueMetrics(queueName: $queueName, timeRange: $timeRange) {
|
||||||
|
queueName
|
||||||
|
timeRange
|
||||||
|
workers
|
||||||
|
details {
|
||||||
|
failed
|
||||||
|
completed
|
||||||
|
waiting
|
||||||
|
active
|
||||||
|
delayed
|
||||||
|
failureRate
|
||||||
|
}
|
||||||
|
data {
|
||||||
|
id
|
||||||
|
data {
|
||||||
|
x
|
||||||
|
y
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`;
|
||||||
@ -2,6 +2,7 @@ import { SettingsAdminHealthStatusRightContainer } from '@/settings/admin-panel/
|
|||||||
import { SettingsAdminIndicatorHealthStatusContent } from '@/settings/admin-panel/health-status/components/SettingsAdminIndicatorHealthStatusContent';
|
import { SettingsAdminIndicatorHealthStatusContent } from '@/settings/admin-panel/health-status/components/SettingsAdminIndicatorHealthStatusContent';
|
||||||
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
|
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
|
||||||
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
|
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
|
||||||
|
import { SettingsSkeletonLoader } from '@/settings/components/SettingsSkeletonLoader';
|
||||||
import { SettingsPath } from '@/types/SettingsPath';
|
import { SettingsPath } from '@/types/SettingsPath';
|
||||||
import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer';
|
import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer';
|
||||||
import styled from '@emotion/styled';
|
import styled from '@emotion/styled';
|
||||||
@ -22,12 +23,17 @@ const StyledH2Title = styled(H2Title)`
|
|||||||
export const SettingsAdminIndicatorHealthStatus = () => {
|
export const SettingsAdminIndicatorHealthStatus = () => {
|
||||||
const { t } = useLingui();
|
const { t } = useLingui();
|
||||||
const { indicatorId } = useParams();
|
const { indicatorId } = useParams();
|
||||||
const { data, loading } = useGetIndicatorHealthStatusQuery({
|
const { data, loading: loadingIndicatorHealthStatus } =
|
||||||
variables: {
|
useGetIndicatorHealthStatusQuery({
|
||||||
indicatorId: indicatorId as HealthIndicatorId,
|
variables: {
|
||||||
},
|
indicatorId: indicatorId as HealthIndicatorId,
|
||||||
fetchPolicy: 'network-only',
|
},
|
||||||
});
|
fetchPolicy: 'network-only',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (loadingIndicatorHealthStatus) {
|
||||||
|
return <SettingsSkeletonLoader />;
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<SubMenuTopBarContainer
|
<SubMenuTopBarContainer
|
||||||
@ -60,7 +66,6 @@ export const SettingsAdminIndicatorHealthStatus = () => {
|
|||||||
details: data?.getIndicatorHealthStatus?.details,
|
details: data?.getIndicatorHealthStatus?.details,
|
||||||
queues: data?.getIndicatorHealthStatus?.queues,
|
queues: data?.getIndicatorHealthStatus?.queues,
|
||||||
},
|
},
|
||||||
loading: loading,
|
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<Section>
|
<Section>
|
||||||
|
|||||||
@ -1,15 +1,24 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
|
|
||||||
|
import { Queue } from 'bullmq';
|
||||||
|
import { Redis } from 'ioredis';
|
||||||
|
|
||||||
import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service';
|
import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service';
|
||||||
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
||||||
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||||
|
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||||
|
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
|
||||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||||
|
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||||
|
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||||
|
|
||||||
|
jest.mock('bullmq');
|
||||||
|
|
||||||
describe('AdminPanelHealthService', () => {
|
describe('AdminPanelHealthService', () => {
|
||||||
let service: AdminPanelHealthService;
|
let service: AdminPanelHealthService;
|
||||||
@ -17,12 +26,25 @@ describe('AdminPanelHealthService', () => {
|
|||||||
let redisHealth: jest.Mocked<RedisHealthIndicator>;
|
let redisHealth: jest.Mocked<RedisHealthIndicator>;
|
||||||
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
|
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
|
||||||
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
|
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
|
||||||
|
let redisClient: jest.Mocked<RedisClientService>;
|
||||||
|
let environmentService: jest.Mocked<EnvironmentService>;
|
||||||
|
let loggerSpy: jest.SpyInstance;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
databaseHealth = { isHealthy: jest.fn() } as any;
|
databaseHealth = { isHealthy: jest.fn() } as any;
|
||||||
redisHealth = { isHealthy: jest.fn() } as any;
|
redisHealth = { isHealthy: jest.fn() } as any;
|
||||||
workerHealth = { isHealthy: jest.fn() } as any;
|
workerHealth = { isHealthy: jest.fn(), getQueueDetails: jest.fn() } as any;
|
||||||
connectedAccountHealth = { isHealthy: jest.fn() } as any;
|
connectedAccountHealth = { isHealthy: jest.fn() } as any;
|
||||||
|
redisClient = {
|
||||||
|
getClient: jest.fn().mockReturnValue({} as Redis),
|
||||||
|
} as any;
|
||||||
|
environmentService = { get: jest.fn() } as any;
|
||||||
|
|
||||||
|
(Queue as unknown as jest.Mock) = jest.fn().mockImplementation(() => ({
|
||||||
|
getMetrics: jest.fn(),
|
||||||
|
getWorkers: jest.fn(),
|
||||||
|
close: jest.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
providers: [
|
providers: [
|
||||||
@ -31,10 +53,21 @@ describe('AdminPanelHealthService', () => {
|
|||||||
{ provide: RedisHealthIndicator, useValue: redisHealth },
|
{ provide: RedisHealthIndicator, useValue: redisHealth },
|
||||||
{ provide: WorkerHealthIndicator, useValue: workerHealth },
|
{ provide: WorkerHealthIndicator, useValue: workerHealth },
|
||||||
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
|
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
|
||||||
|
{ provide: RedisClientService, useValue: redisClient },
|
||||||
|
{ provide: EnvironmentService, useValue: environmentService },
|
||||||
],
|
],
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
service = module.get<AdminPanelHealthService>(AdminPanelHealthService);
|
service = module.get<AdminPanelHealthService>(AdminPanelHealthService);
|
||||||
|
|
||||||
|
loggerSpy = jest
|
||||||
|
.spyOn(service['logger'], 'error')
|
||||||
|
.mockImplementation(() => {});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
jest.clearAllMocks();
|
||||||
|
loggerSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should be defined', () => {
|
it('should be defined', () => {
|
||||||
@ -62,8 +95,9 @@ describe('AdminPanelHealthService', () => {
|
|||||||
delayed: 4,
|
delayed: 4,
|
||||||
failed: 3,
|
failed: 3,
|
||||||
waiting: 0,
|
waiting: 0,
|
||||||
prioritized: 0,
|
failureRate: 0.3,
|
||||||
},
|
},
|
||||||
|
status: 'up',
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@ -209,26 +243,12 @@ describe('AdminPanelHealthService', () => {
|
|||||||
{
|
{
|
||||||
queueName: 'queue1',
|
queueName: 'queue1',
|
||||||
workers: 2,
|
workers: 2,
|
||||||
metrics: {
|
status: 'up',
|
||||||
active: 1,
|
|
||||||
completed: 10,
|
|
||||||
delayed: 0,
|
|
||||||
failed: 2,
|
|
||||||
waiting: 5,
|
|
||||||
prioritized: 1,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
queueName: 'queue2',
|
queueName: 'queue2',
|
||||||
workers: 0,
|
workers: 0,
|
||||||
metrics: {
|
status: 'up',
|
||||||
active: 0,
|
|
||||||
completed: 5,
|
|
||||||
delayed: 0,
|
|
||||||
failed: 1,
|
|
||||||
waiting: 2,
|
|
||||||
prioritized: 0,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@ -248,8 +268,8 @@ describe('AdminPanelHealthService', () => {
|
|||||||
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
status: AdminPanelHealthServiceStatus.OPERATIONAL,
|
||||||
details: undefined,
|
details: undefined,
|
||||||
queues: mockQueues.map((queue) => ({
|
queues: mockQueues.map((queue) => ({
|
||||||
...queue,
|
|
||||||
id: `worker-${queue.queueName}`,
|
id: `worker-${queue.queueName}`,
|
||||||
|
queueName: queue.queueName,
|
||||||
status:
|
status:
|
||||||
queue.workers > 0
|
queue.workers > 0
|
||||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||||
@ -281,4 +301,305 @@ describe('AdminPanelHealthService', () => {
|
|||||||
).rejects.toThrow('Health indicator not found: invalid');
|
).rejects.toThrow('Health indicator not found: invalid');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('getQueueMetrics', () => {
|
||||||
|
const mockQueue = {
|
||||||
|
getMetrics: jest.fn(),
|
||||||
|
getWorkers: jest.fn(),
|
||||||
|
close: jest.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
jest.clearAllMocks();
|
||||||
|
redisClient.getClient.mockReturnValue({} as Redis);
|
||||||
|
(Queue as unknown as jest.Mock).mockImplementation(() => mockQueue);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return metrics data for a queue with correct data transformation', async () => {
|
||||||
|
const mockCompletedData = Array(240)
|
||||||
|
.fill(0)
|
||||||
|
.map((_, i) => i);
|
||||||
|
const mockFailedData = Array(240)
|
||||||
|
.fill(0)
|
||||||
|
.map((_, i) => i * 0.1);
|
||||||
|
|
||||||
|
workerHealth.getQueueDetails.mockResolvedValue({
|
||||||
|
queueName: 'test-queue',
|
||||||
|
workers: 1,
|
||||||
|
status: 'up',
|
||||||
|
metrics: {
|
||||||
|
active: 1,
|
||||||
|
completed: 30,
|
||||||
|
failed: 3,
|
||||||
|
waiting: 0,
|
||||||
|
delayed: 0,
|
||||||
|
failureRate: 9.1,
|
||||||
|
completedData: mockCompletedData,
|
||||||
|
failedData: mockFailedData,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.getQueueMetrics(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
QueueMetricsTimeRange.FourHours,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
queueName: MessageQueue.messagingQueue,
|
||||||
|
timeRange: QueueMetricsTimeRange.FourHours,
|
||||||
|
workers: 1,
|
||||||
|
details: expect.any(Object),
|
||||||
|
data: expect.arrayContaining([
|
||||||
|
expect.objectContaining({
|
||||||
|
id: 'Completed Jobs',
|
||||||
|
data: expect.arrayContaining([
|
||||||
|
expect.objectContaining({
|
||||||
|
x: expect.any(Number),
|
||||||
|
y: expect.any(Number),
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
}),
|
||||||
|
expect.objectContaining({
|
||||||
|
id: 'Failed Jobs',
|
||||||
|
data: expect.arrayContaining([
|
||||||
|
expect.objectContaining({
|
||||||
|
x: expect.any(Number),
|
||||||
|
y: expect.any(Number),
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle empty metrics data', async () => {
|
||||||
|
workerHealth.getQueueDetails.mockResolvedValue(null);
|
||||||
|
|
||||||
|
const result = await service.getQueueMetrics(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
QueueMetricsTimeRange.FourHours,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.data).toHaveLength(2);
|
||||||
|
expect(result.data[0].data).toHaveLength(240);
|
||||||
|
expect(result.data[1].data).toHaveLength(240);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle metrics service errors', async () => {
|
||||||
|
workerHealth.getQueueDetails.mockRejectedValue(
|
||||||
|
new Error('Metrics error'),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.getQueueMetrics(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
QueueMetricsTimeRange.FourHours,
|
||||||
|
),
|
||||||
|
).rejects.toThrow('Metrics error');
|
||||||
|
|
||||||
|
expect(loggerSpy).toHaveBeenCalledWith(
|
||||||
|
'Error getting metrics for messaging-queue: Metrics error',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('backfilling behavior', () => {
|
||||||
|
it('should handle partial data with correct historical backfilling', async () => {
|
||||||
|
// Test with 40 recent points for 4-hour range (needs 240 points)
|
||||||
|
const partialData = Array(40)
|
||||||
|
.fill(0)
|
||||||
|
.map((_, i) => i + 1);
|
||||||
|
|
||||||
|
workerHealth.getQueueDetails.mockResolvedValue({
|
||||||
|
queueName: 'test-queue',
|
||||||
|
workers: 1,
|
||||||
|
status: 'up',
|
||||||
|
metrics: {
|
||||||
|
failed: 0,
|
||||||
|
completed: 0,
|
||||||
|
waiting: 0,
|
||||||
|
active: 0,
|
||||||
|
delayed: 0,
|
||||||
|
failureRate: 0,
|
||||||
|
completedData: partialData,
|
||||||
|
failedData: partialData,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.getQueueMetrics(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
QueueMetricsTimeRange.FourHours,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Should have 240 total points
|
||||||
|
expect(result.data[0].data).toHaveLength(240);
|
||||||
|
|
||||||
|
// First 200 points should be zero (historical backfill)
|
||||||
|
const historicalPoints = result.data[0].data.slice(0, 200);
|
||||||
|
|
||||||
|
expect(historicalPoints.every((point) => point.y === 0)).toBe(true);
|
||||||
|
|
||||||
|
// Last 40 points should be actual data
|
||||||
|
const actualDataPoints = result.data[0].data.slice(200);
|
||||||
|
|
||||||
|
expect(actualDataPoints.every((point) => point.y > 0)).toBe(true);
|
||||||
|
|
||||||
|
// Verify chronological order (increasing values)
|
||||||
|
const nonZeroValues = actualDataPoints.map((point) => point.y);
|
||||||
|
|
||||||
|
for (let i = 1; i < nonZeroValues.length; i++) {
|
||||||
|
expect(nonZeroValues[i]).toBeGreaterThan(nonZeroValues[i - 1]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle completely empty data with full backfilling', async () => {
|
||||||
|
workerHealth.getQueueDetails.mockResolvedValue({
|
||||||
|
queueName: 'test-queue',
|
||||||
|
workers: 1,
|
||||||
|
status: 'up',
|
||||||
|
metrics: {
|
||||||
|
failed: 0,
|
||||||
|
completed: 0,
|
||||||
|
waiting: 0,
|
||||||
|
active: 0,
|
||||||
|
delayed: 0,
|
||||||
|
failureRate: 0,
|
||||||
|
completedData: [],
|
||||||
|
failedData: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.getQueueMetrics(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
QueueMetricsTimeRange.OneHour,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Should have 60 points for one hour
|
||||||
|
expect(result.data[0].data).toHaveLength(60);
|
||||||
|
// All points should be zero
|
||||||
|
expect(result.data[0].data.every((point) => point.y === 0)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('sampling behavior', () => {
|
||||||
|
it('should correctly sample data for different time ranges', async () => {
|
||||||
|
const testCases = [
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.OneHour,
|
||||||
|
expectedPoints: 60,
|
||||||
|
samplingFactor: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.FourHours,
|
||||||
|
expectedPoints: 240,
|
||||||
|
samplingFactor: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.OneDay,
|
||||||
|
expectedPoints: 240,
|
||||||
|
samplingFactor: 6,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const testCase of testCases) {
|
||||||
|
// Create test data with non-zero values
|
||||||
|
const testData = Array(testCase.expectedPoints * 2)
|
||||||
|
.fill(0)
|
||||||
|
.map((_, i) => i + 1); // Start from 1 to avoid zero values
|
||||||
|
|
||||||
|
workerHealth.getQueueDetails.mockResolvedValue({
|
||||||
|
queueName: 'test-queue',
|
||||||
|
workers: 1,
|
||||||
|
status: 'up',
|
||||||
|
metrics: {
|
||||||
|
failed: 0,
|
||||||
|
completed: 0,
|
||||||
|
waiting: 0,
|
||||||
|
active: 0,
|
||||||
|
delayed: 0,
|
||||||
|
failureRate: 0,
|
||||||
|
completedData: testData,
|
||||||
|
failedData: testData,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.getQueueMetrics(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
testCase.timeRange,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.data[0].data).toHaveLength(testCase.expectedPoints);
|
||||||
|
|
||||||
|
if (testCase.samplingFactor > 1) {
|
||||||
|
const sampledData = result.data[0].data;
|
||||||
|
|
||||||
|
for (let i = 0; i < sampledData.length; i++) {
|
||||||
|
const start = i * testCase.samplingFactor;
|
||||||
|
const end = start + testCase.samplingFactor;
|
||||||
|
const originalDataSlice = testData.slice(start, end);
|
||||||
|
|
||||||
|
if (originalDataSlice.length > 0) {
|
||||||
|
// Add this check
|
||||||
|
const maxInSlice = Math.max(...originalDataSlice);
|
||||||
|
|
||||||
|
expect(sampledData[i].y).toBeLessThanOrEqual(maxInSlice);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getPointsConfiguration', () => {
|
||||||
|
const testCases = [
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.OneHour,
|
||||||
|
expected: {
|
||||||
|
pointsNeeded: 60,
|
||||||
|
samplingFactor: 1,
|
||||||
|
targetVisualizationPoints: 240,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.FourHours,
|
||||||
|
expected: {
|
||||||
|
pointsNeeded: 240,
|
||||||
|
samplingFactor: 1,
|
||||||
|
targetVisualizationPoints: 240,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.TwelveHours,
|
||||||
|
expected: {
|
||||||
|
pointsNeeded: 720,
|
||||||
|
samplingFactor: 3,
|
||||||
|
targetVisualizationPoints: 240,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.OneDay,
|
||||||
|
expected: {
|
||||||
|
pointsNeeded: 1440,
|
||||||
|
samplingFactor: 6,
|
||||||
|
targetVisualizationPoints: 240,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timeRange: QueueMetricsTimeRange.SevenDays,
|
||||||
|
expected: {
|
||||||
|
pointsNeeded: 10080,
|
||||||
|
samplingFactor: 42,
|
||||||
|
targetVisualizationPoints: 240,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
testCases.forEach(({ timeRange, expected }) => {
|
||||||
|
it(`should return correct parameters for ${timeRange}`, () => {
|
||||||
|
const result = service['getPointsConfiguration'](timeRange as any);
|
||||||
|
|
||||||
|
expect(result).toEqual(expected);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -1,23 +1,33 @@
|
|||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
import { HealthIndicatorResult, HealthIndicatorStatus } from '@nestjs/terminus';
|
import { HealthIndicatorResult, HealthIndicatorStatus } from '@nestjs/terminus';
|
||||||
|
|
||||||
|
import { Queue } from 'bullmq';
|
||||||
|
|
||||||
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
|
||||||
import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto';
|
import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto';
|
||||||
|
import { QueueMetricsData } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-data.dto';
|
||||||
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||||
|
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||||
|
import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
||||||
|
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||||
|
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class AdminPanelHealthService {
|
export class AdminPanelHealthService {
|
||||||
|
private readonly logger = new Logger(AdminPanelHealthService.name);
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly databaseHealth: DatabaseHealthIndicator,
|
private readonly databaseHealth: DatabaseHealthIndicator,
|
||||||
private readonly redisHealth: RedisHealthIndicator,
|
private readonly redisHealth: RedisHealthIndicator,
|
||||||
private readonly workerHealth: WorkerHealthIndicator,
|
private readonly workerHealth: WorkerHealthIndicator,
|
||||||
private readonly connectedAccountHealth: ConnectedAccountHealth,
|
private readonly connectedAccountHealth: ConnectedAccountHealth,
|
||||||
|
private readonly redisClient: RedisClientService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
private readonly healthIndicators = {
|
private readonly healthIndicators = {
|
||||||
@ -93,8 +103,8 @@ export class AdminPanelHealthService {
|
|||||||
return {
|
return {
|
||||||
...indicatorStatus,
|
...indicatorStatus,
|
||||||
queues: (indicatorStatus?.queues ?? []).map((queue) => ({
|
queues: (indicatorStatus?.queues ?? []).map((queue) => ({
|
||||||
...queue,
|
|
||||||
id: `${indicatorId}-${queue.queueName}`,
|
id: `${indicatorId}-${queue.queueName}`,
|
||||||
|
queueName: queue.queueName,
|
||||||
status:
|
status:
|
||||||
queue.workers > 0
|
queue.workers > 0
|
||||||
? AdminPanelHealthServiceStatus.OPERATIONAL
|
? AdminPanelHealthServiceStatus.OPERATIONAL
|
||||||
@ -144,4 +154,166 @@ export class AdminPanelHealthService {
|
|||||||
],
|
],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getQueueMetrics(
|
||||||
|
queueName: MessageQueue,
|
||||||
|
timeRange: QueueMetricsTimeRange = QueueMetricsTimeRange.OneDay,
|
||||||
|
): Promise<QueueMetricsData> {
|
||||||
|
const redis = this.redisClient.getClient();
|
||||||
|
const queue = new Queue(queueName, { connection: redis });
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { pointsNeeded, samplingFactor } =
|
||||||
|
this.getPointsConfiguration(timeRange);
|
||||||
|
|
||||||
|
const queueDetails = await this.workerHealth.getQueueDetails(queueName, {
|
||||||
|
pointsNeeded,
|
||||||
|
});
|
||||||
|
|
||||||
|
const completedMetricsArray = queueDetails?.metrics?.completedData;
|
||||||
|
const failedMetricsArray = queueDetails?.metrics?.failedData;
|
||||||
|
|
||||||
|
const completedMetrics = this.extractMetricsData(
|
||||||
|
completedMetricsArray,
|
||||||
|
pointsNeeded,
|
||||||
|
samplingFactor,
|
||||||
|
);
|
||||||
|
|
||||||
|
const failedMetrics = this.extractMetricsData(
|
||||||
|
failedMetricsArray,
|
||||||
|
pointsNeeded,
|
||||||
|
samplingFactor,
|
||||||
|
);
|
||||||
|
|
||||||
|
return this.transformMetricsForGraph(
|
||||||
|
completedMetrics,
|
||||||
|
failedMetrics,
|
||||||
|
timeRange,
|
||||||
|
queueName,
|
||||||
|
queueDetails,
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
`Error getting metrics for ${queueName}: ${error.message}`,
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
await queue.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private getPointsConfiguration(timeRange: QueueMetricsTimeRange): {
|
||||||
|
pointsNeeded: number;
|
||||||
|
samplingFactor: number;
|
||||||
|
targetVisualizationPoints: number;
|
||||||
|
} {
|
||||||
|
const targetVisualizationPoints = 240;
|
||||||
|
|
||||||
|
let pointsNeeded: number;
|
||||||
|
|
||||||
|
switch (timeRange) {
|
||||||
|
case QueueMetricsTimeRange.OneHour:
|
||||||
|
pointsNeeded = 60; // 60 points (1 hour)
|
||||||
|
break;
|
||||||
|
case QueueMetricsTimeRange.FourHours:
|
||||||
|
pointsNeeded = 4 * 60; // 240 points (4 hours)
|
||||||
|
break;
|
||||||
|
case QueueMetricsTimeRange.TwelveHours:
|
||||||
|
pointsNeeded = 12 * 60; // 720 points (12 hours)
|
||||||
|
break;
|
||||||
|
case QueueMetricsTimeRange.OneDay:
|
||||||
|
pointsNeeded = 24 * 60; // 1440 points (24 hours)
|
||||||
|
break;
|
||||||
|
case QueueMetricsTimeRange.SevenDays:
|
||||||
|
pointsNeeded = 7 * 24 * 60; // 10080 points (7 days)
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
pointsNeeded = 24 * 60; // Default to 1 day
|
||||||
|
}
|
||||||
|
|
||||||
|
const samplingFactor =
|
||||||
|
pointsNeeded <= targetVisualizationPoints
|
||||||
|
? 1
|
||||||
|
: Math.ceil(pointsNeeded / targetVisualizationPoints);
|
||||||
|
|
||||||
|
return {
|
||||||
|
pointsNeeded,
|
||||||
|
samplingFactor,
|
||||||
|
targetVisualizationPoints,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractMetricsData(
|
||||||
|
metrics: number[] | undefined,
|
||||||
|
pointsNeeded: number,
|
||||||
|
samplingFactor = 1,
|
||||||
|
): number[] {
|
||||||
|
if (!metrics || !Array.isArray(metrics)) {
|
||||||
|
return Array(Math.ceil(pointsNeeded / samplingFactor)).fill(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const targetPoints = Math.ceil(pointsNeeded / samplingFactor);
|
||||||
|
|
||||||
|
const relevantData = metrics.slice(-pointsNeeded);
|
||||||
|
const result: number[] = [];
|
||||||
|
|
||||||
|
const backfillCount = Math.max(
|
||||||
|
0,
|
||||||
|
targetPoints - Math.ceil(relevantData.length / samplingFactor),
|
||||||
|
);
|
||||||
|
|
||||||
|
result.push(...Array(backfillCount).fill(0));
|
||||||
|
|
||||||
|
for (let i = 0; i < relevantData.length; i += samplingFactor) {
|
||||||
|
const chunk = relevantData.slice(i, i + samplingFactor);
|
||||||
|
|
||||||
|
result.push(Math.max(...chunk));
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.slice(0, targetPoints);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Error extracting metrics data: ${error.message}`);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private transformMetricsForGraph(
|
||||||
|
completedMetrics: number[],
|
||||||
|
failedMetrics: number[],
|
||||||
|
timeRange: QueueMetricsTimeRange,
|
||||||
|
queueName: MessageQueue,
|
||||||
|
queueDetails: WorkerQueueHealth | null,
|
||||||
|
): QueueMetricsData {
|
||||||
|
try {
|
||||||
|
return {
|
||||||
|
queueName,
|
||||||
|
timeRange,
|
||||||
|
details: queueDetails?.metrics ?? null,
|
||||||
|
workers: queueDetails?.workers ?? 0,
|
||||||
|
data: [
|
||||||
|
{
|
||||||
|
id: 'Completed Jobs',
|
||||||
|
data: completedMetrics.map((count, index) => ({
|
||||||
|
x: index,
|
||||||
|
y: count,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'Failed Jobs',
|
||||||
|
data: failedMetrics.map((count, index) => ({
|
||||||
|
x: index,
|
||||||
|
y: count,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
`Error transforming metrics for graph: ${error.message}`,
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -10,20 +10,24 @@ import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-he
|
|||||||
import { UpdateWorkspaceFeatureFlagInput } from 'src/engine/core-modules/admin-panel/dtos/update-workspace-feature-flag.input';
|
import { UpdateWorkspaceFeatureFlagInput } from 'src/engine/core-modules/admin-panel/dtos/update-workspace-feature-flag.input';
|
||||||
import { UserLookup } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.entity';
|
import { UserLookup } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.entity';
|
||||||
import { UserLookupInput } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.input';
|
import { UserLookupInput } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.input';
|
||||||
|
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||||
import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-graphql-api-exception.filter';
|
import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-graphql-api-exception.filter';
|
||||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||||
|
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||||
|
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||||
import { ImpersonateGuard } from 'src/engine/guards/impersonate-guard';
|
import { ImpersonateGuard } from 'src/engine/guards/impersonate-guard';
|
||||||
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
|
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
|
||||||
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
||||||
|
|
||||||
import { AdminPanelHealthServiceData } from './dtos/admin-panel-health-service-data.dto';
|
import { AdminPanelHealthServiceData } from './dtos/admin-panel-health-service-data.dto';
|
||||||
|
import { QueueMetricsData } from './dtos/queue-metrics-data.dto';
|
||||||
@Resolver()
|
@Resolver()
|
||||||
@UseFilters(AuthGraphqlApiExceptionFilter)
|
@UseFilters(AuthGraphqlApiExceptionFilter)
|
||||||
export class AdminPanelResolver {
|
export class AdminPanelResolver {
|
||||||
constructor(
|
constructor(
|
||||||
private adminService: AdminPanelService,
|
private adminService: AdminPanelService,
|
||||||
private adminPanelHealthService: AdminPanelHealthService,
|
private adminPanelHealthService: AdminPanelHealthService,
|
||||||
|
private workerHealthIndicator: WorkerHealthIndicator,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard)
|
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard)
|
||||||
@ -68,6 +72,7 @@ export class AdminPanelResolver {
|
|||||||
return this.adminPanelHealthService.getSystemHealthStatus();
|
return this.adminPanelHealthService.getSystemHealthStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard)
|
||||||
@Query(() => AdminPanelHealthServiceData)
|
@Query(() => AdminPanelHealthServiceData)
|
||||||
async getIndicatorHealthStatus(
|
async getIndicatorHealthStatus(
|
||||||
@Args('indicatorId', {
|
@Args('indicatorId', {
|
||||||
@ -77,4 +82,22 @@ export class AdminPanelResolver {
|
|||||||
): Promise<AdminPanelHealthServiceData> {
|
): Promise<AdminPanelHealthServiceData> {
|
||||||
return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorId);
|
return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, ImpersonateGuard)
|
||||||
|
@Query(() => QueueMetricsData)
|
||||||
|
async getQueueMetrics(
|
||||||
|
@Args('queueName', { type: () => String })
|
||||||
|
queueName: string,
|
||||||
|
@Args('timeRange', {
|
||||||
|
nullable: true,
|
||||||
|
defaultValue: QueueMetricsTimeRange.OneDay,
|
||||||
|
type: () => QueueMetricsTimeRange,
|
||||||
|
})
|
||||||
|
timeRange: QueueMetricsTimeRange = QueueMetricsTimeRange.OneHour,
|
||||||
|
): Promise<QueueMetricsData> {
|
||||||
|
return await this.adminPanelHealthService.getQueueMetrics(
|
||||||
|
queueName as MessageQueue,
|
||||||
|
timeRange,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,13 +1,15 @@
|
|||||||
import { Field, ObjectType } from '@nestjs/graphql';
|
import { Field, ObjectType } from '@nestjs/graphql';
|
||||||
|
|
||||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||||
import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
|
||||||
|
|
||||||
@ObjectType()
|
@ObjectType()
|
||||||
export class AdminPanelWorkerQueueHealth extends WorkerQueueHealth {
|
export class AdminPanelWorkerQueueHealth {
|
||||||
@Field(() => String)
|
@Field(() => String)
|
||||||
id: string;
|
id: string;
|
||||||
|
|
||||||
|
@Field(() => String)
|
||||||
|
queueName: string;
|
||||||
|
|
||||||
@Field(() => AdminPanelHealthServiceStatus)
|
@Field(() => AdminPanelHealthServiceStatus)
|
||||||
status: AdminPanelHealthServiceStatus;
|
status: AdminPanelHealthServiceStatus;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,10 @@
|
|||||||
|
import { Field, ObjectType } from '@nestjs/graphql';
|
||||||
|
|
||||||
|
@ObjectType()
|
||||||
|
export class QueueMetricsDataPoint {
|
||||||
|
@Field(() => Number)
|
||||||
|
x: number;
|
||||||
|
|
||||||
|
@Field(() => Number)
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
import { Field, ObjectType } from '@nestjs/graphql';
|
||||||
|
|
||||||
|
import { QueueMetricsSeries } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-series.dto';
|
||||||
|
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||||
|
import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-queue-metrics.type';
|
||||||
|
|
||||||
|
@ObjectType()
|
||||||
|
export class QueueMetricsData {
|
||||||
|
@Field(() => String)
|
||||||
|
queueName: string;
|
||||||
|
|
||||||
|
@Field(() => Number)
|
||||||
|
workers: number;
|
||||||
|
|
||||||
|
@Field(() => QueueMetricsTimeRange)
|
||||||
|
timeRange: QueueMetricsTimeRange;
|
||||||
|
|
||||||
|
@Field(() => WorkerQueueMetrics, { nullable: true })
|
||||||
|
details: WorkerQueueMetrics | null;
|
||||||
|
|
||||||
|
@Field(() => [QueueMetricsSeries])
|
||||||
|
data: QueueMetricsSeries[];
|
||||||
|
}
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
import { Field, ObjectType } from '@nestjs/graphql';
|
||||||
|
|
||||||
|
import { QueueMetricsDataPoint } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-data-point.dto';
|
||||||
|
|
||||||
|
@ObjectType()
|
||||||
|
export class QueueMetricsSeries {
|
||||||
|
@Field()
|
||||||
|
id: string;
|
||||||
|
|
||||||
|
@Field(() => [QueueMetricsDataPoint])
|
||||||
|
data: QueueMetricsDataPoint[];
|
||||||
|
}
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
import { registerEnumType } from '@nestjs/graphql';
|
||||||
|
|
||||||
|
export enum QueueMetricsTimeRange {
|
||||||
|
SevenDays = '7D',
|
||||||
|
OneDay = '1D',
|
||||||
|
TwelveHours = '12H',
|
||||||
|
FourHours = '4H',
|
||||||
|
OneHour = '1H',
|
||||||
|
}
|
||||||
|
|
||||||
|
registerEnumType(QueueMetricsTimeRange, {
|
||||||
|
name: 'QueueMetricsTimeRange',
|
||||||
|
});
|
||||||
@ -12,12 +12,10 @@ import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-c
|
|||||||
const mockQueueInstance = {
|
const mockQueueInstance = {
|
||||||
getWorkers: jest.fn().mockResolvedValue([]),
|
getWorkers: jest.fn().mockResolvedValue([]),
|
||||||
close: jest.fn().mockResolvedValue(undefined),
|
close: jest.fn().mockResolvedValue(undefined),
|
||||||
getFailedCount: jest.fn().mockResolvedValue(0),
|
getMetrics: jest.fn().mockResolvedValue({ count: 0, data: [] }),
|
||||||
getCompletedCount: jest.fn().mockResolvedValue(0),
|
|
||||||
getWaitingCount: jest.fn().mockResolvedValue(0),
|
getWaitingCount: jest.fn().mockResolvedValue(0),
|
||||||
getActiveCount: jest.fn().mockResolvedValue(0),
|
getActiveCount: jest.fn().mockResolvedValue(0),
|
||||||
getDelayedCount: jest.fn().mockResolvedValue(0),
|
getDelayedCount: jest.fn().mockResolvedValue(0),
|
||||||
getPrioritizedCount: jest.fn().mockResolvedValue(0),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
jest.mock('bullmq', () => ({
|
jest.mock('bullmq', () => ({
|
||||||
@ -28,6 +26,7 @@ describe('WorkerHealthIndicator', () => {
|
|||||||
let service: WorkerHealthIndicator;
|
let service: WorkerHealthIndicator;
|
||||||
let mockRedis: jest.Mocked<Pick<Redis, 'ping'>>;
|
let mockRedis: jest.Mocked<Pick<Redis, 'ping'>>;
|
||||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||||
|
let loggerSpy: jest.SpyInstance;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
mockRedis = {
|
mockRedis = {
|
||||||
@ -64,11 +63,23 @@ describe('WorkerHealthIndicator', () => {
|
|||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
service = module.get<WorkerHealthIndicator>(WorkerHealthIndicator);
|
service = module.get<WorkerHealthIndicator>(WorkerHealthIndicator);
|
||||||
|
|
||||||
|
loggerSpy = jest
|
||||||
|
.spyOn(service['logger'], 'error')
|
||||||
|
.mockImplementation(() => {});
|
||||||
jest.useFakeTimers();
|
jest.useFakeTimers();
|
||||||
|
|
||||||
|
// Reset mocks to their default success state before each test
|
||||||
|
mockQueueInstance.getWorkers.mockResolvedValue([]);
|
||||||
|
mockQueueInstance.getMetrics.mockResolvedValue({ count: 0, data: [] });
|
||||||
|
mockQueueInstance.getWaitingCount.mockResolvedValue(0);
|
||||||
|
mockQueueInstance.getActiveCount.mockResolvedValue(0);
|
||||||
|
mockQueueInstance.getDelayedCount.mockResolvedValue(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
jest.useRealTimers();
|
jest.useRealTimers();
|
||||||
|
jest.clearAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should be defined', () => {
|
it('should be defined', () => {
|
||||||
@ -133,4 +144,204 @@ describe('WorkerHealthIndicator', () => {
|
|||||||
Object.keys(MessageQueue).length,
|
Object.keys(MessageQueue).length,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should return down status when failure rate exceeds threshold', async () => {
|
||||||
|
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||||
|
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||||
|
if (type === 'failed') {
|
||||||
|
return Promise.resolve({ count: 600 });
|
||||||
|
}
|
||||||
|
if (type === 'completed') {
|
||||||
|
return Promise.resolve({ count: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.resolve({ count: 0 });
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.isHealthy();
|
||||||
|
|
||||||
|
expect(result.worker.status).toBe('up');
|
||||||
|
expect('queues' in result.worker).toBe(true);
|
||||||
|
if ('queues' in result.worker) {
|
||||||
|
expect(result.worker.queues[0].status).toBe('down');
|
||||||
|
expect(result.worker.queues[0].metrics).toEqual({
|
||||||
|
failed: 600,
|
||||||
|
completed: 400,
|
||||||
|
waiting: 0,
|
||||||
|
active: 0,
|
||||||
|
delayed: 0,
|
||||||
|
failureRate: 60,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return complete metrics for active workers', async () => {
|
||||||
|
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||||
|
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||||
|
if (type === 'failed') {
|
||||||
|
return Promise.resolve({ count: 10 });
|
||||||
|
}
|
||||||
|
if (type === 'completed') {
|
||||||
|
return Promise.resolve({ count: 90 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.resolve({ count: 0 });
|
||||||
|
});
|
||||||
|
mockQueueInstance.getWaitingCount.mockResolvedValue(5);
|
||||||
|
mockQueueInstance.getActiveCount.mockResolvedValue(2);
|
||||||
|
mockQueueInstance.getDelayedCount.mockResolvedValue(1);
|
||||||
|
|
||||||
|
const result = await service.isHealthy();
|
||||||
|
|
||||||
|
expect(result.worker.status).toBe('up');
|
||||||
|
expect('queues' in result.worker).toBe(true);
|
||||||
|
if ('queues' in result.worker) {
|
||||||
|
expect(result.worker.queues[0].metrics).toEqual({
|
||||||
|
failed: 10,
|
||||||
|
completed: 90,
|
||||||
|
waiting: 5,
|
||||||
|
active: 2,
|
||||||
|
delayed: 1,
|
||||||
|
failureRate: 10,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle queue errors gracefully', async () => {
|
||||||
|
mockQueueInstance.getWorkers.mockRejectedValue(new Error('Queue error'));
|
||||||
|
mockQueueInstance.getMetrics.mockRejectedValue(new Error('Queue error'));
|
||||||
|
mockQueueInstance.getWaitingCount.mockRejectedValue(
|
||||||
|
new Error('Queue error'),
|
||||||
|
);
|
||||||
|
mockQueueInstance.getActiveCount.mockRejectedValue(
|
||||||
|
new Error('Queue error'),
|
||||||
|
);
|
||||||
|
mockQueueInstance.getDelayedCount.mockRejectedValue(
|
||||||
|
new Error('Queue error'),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await service.isHealthy();
|
||||||
|
|
||||||
|
expect(result.worker.status).toBe('down');
|
||||||
|
expect('error' in result.worker).toBe(true);
|
||||||
|
if ('error' in result.worker) {
|
||||||
|
expect(result.worker.error).toBe(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS);
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(loggerSpy).toHaveBeenCalled();
|
||||||
|
Object.values(MessageQueue).forEach((queueName) => {
|
||||||
|
expect(loggerSpy).toHaveBeenCalledWith(
|
||||||
|
`Error getting queue details for ${queueName}: Queue error`,
|
||||||
|
);
|
||||||
|
expect(loggerSpy).toHaveBeenCalledWith(
|
||||||
|
`Error checking worker for queue ${queueName}: Queue error`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getQueueDetails', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
// Reset mocks to clean state before each test in this describe block
|
||||||
|
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||||
|
mockQueueInstance.getMetrics.mockResolvedValue({ count: 0, data: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return metrics with time series data when pointsNeeded is provided', async () => {
|
||||||
|
const pointsNeeded = 60;
|
||||||
|
|
||||||
|
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||||
|
if (type === 'failed') {
|
||||||
|
return Promise.resolve({
|
||||||
|
count: 10,
|
||||||
|
data: Array(pointsNeeded).fill(10 / pointsNeeded),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (type === 'completed') {
|
||||||
|
return Promise.resolve({
|
||||||
|
count: 90,
|
||||||
|
data: Array(pointsNeeded).fill(90 / pointsNeeded),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.resolve({ count: 0, data: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.getQueueDetails(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
{
|
||||||
|
pointsNeeded,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toBeDefined();
|
||||||
|
expect(result?.metrics).toMatchObject({
|
||||||
|
failed: 10,
|
||||||
|
completed: 90,
|
||||||
|
failedData: expect.any(Array),
|
||||||
|
completedData: expect.any(Array),
|
||||||
|
});
|
||||||
|
expect(result?.metrics.failedData).toHaveLength(pointsNeeded);
|
||||||
|
expect(result?.metrics.completedData).toHaveLength(pointsNeeded);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle invalid metrics data gracefully', async () => {
|
||||||
|
const invalidData = ['invalid', null, undefined, '1', 2];
|
||||||
|
|
||||||
|
mockQueueInstance.getMetrics.mockResolvedValue({
|
||||||
|
count: 0,
|
||||||
|
data: invalidData,
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.getQueueDetails(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
{
|
||||||
|
pointsNeeded: 5,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toBeDefined();
|
||||||
|
expect(result?.metrics.failedData).toEqual([NaN, 0, NaN, 1, 2]);
|
||||||
|
expect(result?.metrics.completedData).toEqual([NaN, 0, NaN, 1, 2]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should calculate correct failure rate with time series data', async () => {
|
||||||
|
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||||
|
if (type === 'failed') {
|
||||||
|
return Promise.resolve({ count: 600, data: Array(10).fill(60) });
|
||||||
|
}
|
||||||
|
if (type === 'completed') {
|
||||||
|
return Promise.resolve({ count: 400, data: Array(10).fill(40) });
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.resolve({ count: 0, data: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.getQueueDetails(
|
||||||
|
MessageQueue.messagingQueue,
|
||||||
|
{
|
||||||
|
pointsNeeded: 10,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toBeDefined();
|
||||||
|
expect(result?.metrics).toMatchObject({
|
||||||
|
failed: 600,
|
||||||
|
completed: 400,
|
||||||
|
failureRate: 60,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle queue errors gracefully', async () => {
|
||||||
|
mockQueueInstance.getWorkers.mockRejectedValue(new Error('Queue error'));
|
||||||
|
mockQueueInstance.getMetrics.mockRejectedValue(new Error('Queue error'));
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.getQueueDetails(MessageQueue.messagingQueue),
|
||||||
|
).rejects.toThrow('Queue error');
|
||||||
|
|
||||||
|
expect(loggerSpy).toHaveBeenCalledWith(
|
||||||
|
`Error getting queue details for ${MessageQueue.messagingQueue}: Queue error`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -1,9 +1,13 @@
|
|||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
import {
|
||||||
|
HealthIndicatorResult,
|
||||||
|
HealthIndicatorService,
|
||||||
|
} from '@nestjs/terminus';
|
||||||
|
|
||||||
import { Queue } from 'bullmq';
|
import { Queue } from 'bullmq';
|
||||||
|
|
||||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||||
|
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const';
|
||||||
import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
||||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
|
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
|
||||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||||
@ -11,12 +15,14 @@ import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-c
|
|||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class WorkerHealthIndicator {
|
export class WorkerHealthIndicator {
|
||||||
|
private readonly logger = new Logger(WorkerHealthIndicator.name);
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly redisClient: RedisClientService,
|
private readonly redisClient: RedisClientService,
|
||||||
private readonly healthIndicatorService: HealthIndicatorService,
|
private readonly healthIndicatorService: HealthIndicatorService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async isHealthy() {
|
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||||
const indicator = this.healthIndicatorService.check('worker');
|
const indicator = this.healthIndicatorService.check('worker');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@ -42,51 +48,106 @@ export class WorkerHealthIndicator {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async checkWorkers() {
|
async getQueueDetails(
|
||||||
|
queueName: MessageQueue,
|
||||||
|
options?: {
|
||||||
|
pointsNeeded?: number;
|
||||||
|
},
|
||||||
|
): Promise<WorkerQueueHealth | null> {
|
||||||
const redis = this.redisClient.getClient();
|
const redis = this.redisClient.getClient();
|
||||||
|
const queue = new Queue(queueName, { connection: redis });
|
||||||
|
|
||||||
|
try {
|
||||||
|
const workers = await queue.getWorkers();
|
||||||
|
|
||||||
|
if (workers.length > 0) {
|
||||||
|
const metricsParams = options?.pointsNeeded
|
||||||
|
? [0, options.pointsNeeded - 1]
|
||||||
|
: [];
|
||||||
|
|
||||||
|
const [
|
||||||
|
failedMetrics,
|
||||||
|
completedMetrics,
|
||||||
|
waitingCount,
|
||||||
|
activeCount,
|
||||||
|
delayedCount,
|
||||||
|
] = await Promise.all([
|
||||||
|
queue.getMetrics('failed', ...metricsParams),
|
||||||
|
queue.getMetrics('completed', ...metricsParams),
|
||||||
|
queue.getWaitingCount(),
|
||||||
|
queue.getActiveCount(),
|
||||||
|
queue.getDelayedCount(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const failedCount = options?.pointsNeeded
|
||||||
|
? this.calculateMetricsSum(failedMetrics.data)
|
||||||
|
: failedMetrics.count;
|
||||||
|
|
||||||
|
const completedCount = options?.pointsNeeded
|
||||||
|
? this.calculateMetricsSum(completedMetrics.data)
|
||||||
|
: completedMetrics.count;
|
||||||
|
|
||||||
|
const totalJobs = failedCount + completedCount;
|
||||||
|
const failureRate =
|
||||||
|
totalJobs > 0
|
||||||
|
? Number(((failedCount / totalJobs) * 100).toFixed(1))
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
return {
|
||||||
|
queueName,
|
||||||
|
workers: workers.length,
|
||||||
|
status: failureRate > METRICS_FAILURE_RATE_THRESHOLD ? 'down' : 'up',
|
||||||
|
metrics: {
|
||||||
|
failed: failedCount,
|
||||||
|
completed: completedCount,
|
||||||
|
waiting: waitingCount,
|
||||||
|
active: activeCount,
|
||||||
|
delayed: delayedCount,
|
||||||
|
failureRate,
|
||||||
|
...(options?.pointsNeeded && {
|
||||||
|
failedData: failedMetrics.data.map(Number),
|
||||||
|
completedData: completedMetrics.data.map(Number),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
`Error getting queue details for ${queueName}: ${error.message}`,
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
await queue.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private calculateMetricsSum(data: string[] | number[]): number {
|
||||||
|
const sum = data.reduce((sum: number, value: string | number) => {
|
||||||
|
const numericValue = Number(value);
|
||||||
|
|
||||||
|
return sum + (isNaN(numericValue) ? 0 : numericValue);
|
||||||
|
}, 0);
|
||||||
|
|
||||||
|
return Math.round(Number(sum));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async checkWorkers() {
|
||||||
const queues = Object.values(MessageQueue);
|
const queues = Object.values(MessageQueue);
|
||||||
const queueStatuses: WorkerQueueHealth[] = [];
|
const queueStatuses: WorkerQueueHealth[] = [];
|
||||||
|
|
||||||
for (const queueName of queues) {
|
for (const queueName of queues) {
|
||||||
const queue = new Queue(queueName, { connection: redis });
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const workers = await queue.getWorkers();
|
const queueDetails = await this.getQueueDetails(queueName);
|
||||||
|
|
||||||
if (workers.length > 0) {
|
if (queueDetails) {
|
||||||
const [
|
queueStatuses.push(queueDetails);
|
||||||
failedCount,
|
|
||||||
completedCount,
|
|
||||||
waitingCount,
|
|
||||||
activeCount,
|
|
||||||
delayedCount,
|
|
||||||
prioritizedCount,
|
|
||||||
] = await Promise.all([
|
|
||||||
queue.getFailedCount(),
|
|
||||||
queue.getCompletedCount(),
|
|
||||||
queue.getWaitingCount(),
|
|
||||||
queue.getActiveCount(),
|
|
||||||
queue.getDelayedCount(),
|
|
||||||
queue.getPrioritizedCount(),
|
|
||||||
]);
|
|
||||||
|
|
||||||
queueStatuses.push({
|
|
||||||
queueName: queueName,
|
|
||||||
workers: workers.length,
|
|
||||||
metrics: {
|
|
||||||
failed: failedCount,
|
|
||||||
completed: completedCount,
|
|
||||||
waiting: waitingCount,
|
|
||||||
active: activeCount,
|
|
||||||
delayed: delayedCount,
|
|
||||||
prioritized: prioritizedCount,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await queue.close();
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await queue.close();
|
this.logger.error(
|
||||||
|
`Error checking worker for queue ${queueName}: ${error.message}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -7,6 +7,9 @@ export class WorkerQueueHealth {
|
|||||||
@Field(() => String)
|
@Field(() => String)
|
||||||
queueName: string;
|
queueName: string;
|
||||||
|
|
||||||
|
@Field(() => String)
|
||||||
|
status: string;
|
||||||
|
|
||||||
@Field(() => Number)
|
@Field(() => Number)
|
||||||
workers: number;
|
workers: number;
|
||||||
|
|
||||||
|
|||||||
@ -18,5 +18,11 @@ export class WorkerQueueMetrics {
|
|||||||
delayed: number;
|
delayed: number;
|
||||||
|
|
||||||
@Field(() => Number)
|
@Field(() => Number)
|
||||||
prioritized: number;
|
failureRate: number;
|
||||||
|
|
||||||
|
@Field(() => [Number], { nullable: true })
|
||||||
|
failedData?: number[];
|
||||||
|
|
||||||
|
@Field(() => [Number], { nullable: true })
|
||||||
|
completedData?: number[];
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
import { OnModuleDestroy } from '@nestjs/common';
|
import { OnModuleDestroy } from '@nestjs/common';
|
||||||
|
|
||||||
import { JobsOptions, Queue, QueueOptions, Worker } from 'bullmq';
|
import { JobsOptions, MetricsTime, Queue, QueueOptions, Worker } from 'bullmq';
|
||||||
import { isDefined } from 'twenty-shared';
|
import { isDefined } from 'twenty-shared';
|
||||||
import { v4 } from 'uuid';
|
import { v4 } from 'uuid';
|
||||||
|
|
||||||
@ -50,12 +50,16 @@ export class BullMQDriver implements MessageQueueDriver, OnModuleDestroy {
|
|||||||
handler: (job: MessageQueueJob<T>) => Promise<void>,
|
handler: (job: MessageQueueJob<T>) => Promise<void>,
|
||||||
options?: MessageQueueWorkerOptions,
|
options?: MessageQueueWorkerOptions,
|
||||||
) {
|
) {
|
||||||
const workerOptions = isDefined(options?.concurrency)
|
const workerOptions = {
|
||||||
? {
|
...this.options,
|
||||||
...this.options,
|
...(isDefined(options?.concurrency)
|
||||||
concurrency: options.concurrency,
|
? { concurrency: options.concurrency }
|
||||||
}
|
: {}),
|
||||||
: this.options;
|
metrics: {
|
||||||
|
maxDataPoints: MetricsTime.ONE_WEEK,
|
||||||
|
collectInterval: 60000,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
this.workerMap[queueName] = new Worker(
|
this.workerMap[queueName] = new Worker(
|
||||||
queueName,
|
queueName,
|
||||||
|
|||||||
Reference in New Issue
Block a user